feat: insert data de-weight;perf: input queue

src/service/errorCode.ts

@@ -24,10 +24,10 @@ export const openaiError: Record<string, string> = {
   'Bad Request': 'Bad Request~ 可能内容太多了',
   'Bad Gateway': '网关异常，请重试'
 };
-export const openaiError2: Record<string, string> = {
-  insufficient_quota: 'API 余额不足',
-  billing_not_active: 'openai 账号异常',
-  invalid_request_error: '无效的 openai 请求'
+export const openaiAccountError: Record<string, string> = {
+  // insufficient_quota: 'API 余额不足',
+  invalid_api_key: 'openai 账号异常'
+  // invalid_request_error: '无效的 openai 请求'
 };
 export const proxyError: Record<string, boolean> = {
   ECONNABORTED: true,

src/service/events/generateQA.ts

@@ -2,7 +2,7 @@ import { TrainingData } from '@/service/mongo';
 import { getApiKey } from '../utils/auth';
 import { OpenAiChatEnum } from '@/constants/model';
 import { pushSplitDataBill } from '@/service/events/pushBill';
-import { openaiError2 } from '../errorCode';
+import { openaiAccountError } from '../errorCode';
 import { modelServiceToolMap } from '../utils/chat';
 import { ChatRoleEnum } from '@/constants/chat';
 import { BillTypeEnum } from '@/constants/user';
@@ -81,8 +81,6 @@ export async function generateQA(): Promise<any> {
       type: 'training'
     });
 
-    console.log(`正在生成一组QA。ID: ${trainingId}`);
-
     const startTime = Date.now();
 
     // 请求 chatgpt 获取回答
@@ -137,7 +135,7 @@ A2:
     const responseList = response.map((item) => item.result).flat();
 
     // 创建 向量生成 队列
-    pushDataToKb({
+    await pushDataToKb({
       kbId,
       data: responseList,
       userId,
@@ -161,8 +159,16 @@ A2:
       console.log('生成QA错误:', err);
     }
 
-    // openai 账号异常或者账号余额不足，删除任务
-    if (openaiError2[err?.response?.data?.error?.type] || err === ERROR_ENUM.insufficientQuota) {
+    // message error or openai account error
+    if (
+      err?.message === 'invalid message format' ||
+      openaiAccountError[err?.response?.data?.error?.code]
+    ) {
+      await TrainingData.findByIdAndRemove(trainingId);
+    }
+
+    // 账号余额不足，删除任务
+    if (err === ERROR_ENUM.insufficientQuota) {
       console.log('余额不足，删除向量生成任务');
       await TrainingData.deleteMany({
         userId

src/service/events/generateVector.ts

@@ -1,4 +1,4 @@
-import { openaiError2 } from '../errorCode';
+import { openaiAccountError } from '../errorCode';
 import { insertKbItem } from '@/service/pg';
 import { openaiEmbedding } from '@/pages/api/openapi/plugin/openaiEmbedding';
 import { TrainingData } from '../models/trainingData';
@@ -111,8 +111,17 @@ export async function generateVector(): Promise<any> {
       console.log('生成向量错误:', err);
     }
 
-    // openai 账号异常或者账号余额不足，删除任务
-    if (openaiError2[err?.response?.data?.error?.type] || err === ERROR_ENUM.insufficientQuota) {
+    // message error or openai account error
+    if (
+      err?.message === 'invalid message format' ||
+      openaiAccountError[err?.response?.data?.error?.code]
+    ) {
+      console.log('删除一个任务');
+      await TrainingData.findByIdAndRemove(trainingId);
+    }
+
+    // 账号余额不足，删除任务
+    if (err === ERROR_ENUM.insufficientQuota) {
       console.log('余额不足，删除向量生成任务');
       await TrainingData.deleteMany({
         userId

src/service/events/pushBill.ts

@@ -134,9 +134,9 @@ export const pushGenerateVectorBill = async ({
   text: string;
   tokenLen: number;
 }) => {
-  console.log(
-    `vector generate success. text len: ${text.length}. token len: ${tokenLen}. pay:${isPay}`
-  );
+  // console.log(
+  //   `vector generate success. text len: ${text.length}. token len: ${tokenLen}. pay:${isPay}`
+  // );
   if (!isPay) return;
 
   let billId;

src/service/pg.ts

@@ -177,8 +177,8 @@ export const insertKbItem = ({
     values: data.map((item) => [
       { key: 'user_id', value: userId },
       { key: 'kb_id', value: kbId },
-      { key: 'q', value: item.q },
-      { key: 'a', value: item.a },
+      { key: 'q', value: item.q.replace(/'/g, '"') },
+      { key: 'a', value: item.a.replace(/'/g, '"') },
       { key: 'vector', value: `[${item.vector}]` }
     ])
   });

src/service/response.ts

@@ -1,5 +1,11 @@
 import { NextApiResponse } from 'next';
-import { openaiError, openaiError2, proxyError, ERROR_RESPONSE, ERROR_ENUM } from './errorCode';
+import {
+  openaiError,
+  openaiAccountError,
+  proxyError,
+  ERROR_RESPONSE,
+  ERROR_ENUM
+} from './errorCode';
 import { clearCookie } from './utils/tools';
 
 export interface ResponseType<T = any> {
@@ -40,8 +46,8 @@ export const jsonRes = <T = any>(
       msg = '接口连接异常';
     } else if (error?.response?.data?.error?.message) {
       msg = error?.response?.data?.error?.message;
-    } else if (openaiError2[error?.response?.data?.error?.type]) {
-      msg = openaiError2[error?.response?.data?.error?.type];
+    } else if (openaiAccountError[error?.response?.data?.error?.code]) {
+      msg = openaiAccountError[error?.response?.data?.error?.code];
     } else if (openaiError[error?.response?.statusText]) {
       msg = openaiError[error.response.statusText];
     }