perf: code

src/service/events/generateQA.ts

@@ -7,7 +7,7 @@ import { modelServiceToolMap } from '../utils/chat';
 import { ChatRoleEnum } from '@/constants/chat';
 import { BillTypeEnum } from '@/constants/user';
 import { pushDataToKb } from '@/pages/api/openapi/kb/pushData';
-import { TrainingTypeEnum } from '@/constants/plugin';
+import { TrainingModeEnum } from '@/constants/plugin';
 import { ERROR_ENUM } from '../errorCode';
 
 export async function generateQA(): Promise<any> {
@@ -23,7 +23,7 @@ export async function generateQA(): Promise<any> {
     // 找出一个需要生成的 dataItem (4分钟锁)
     const data = await TrainingData.findOneAndUpdate(
       {
-        mode: TrainingTypeEnum.qa,
+        mode: TrainingModeEnum.qa,
         lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) }
       },
       {
@@ -115,7 +115,7 @@ A2:
       kbId,
       data: responseList,
       userId,
-      mode: TrainingTypeEnum.index
+      mode: TrainingModeEnum.index
     });
 
     // delete data from training
@@ -126,6 +126,7 @@ A2:
     global.qaQueueLen--;
     generateQA();
   } catch (err: any) {
+    global.qaQueueLen--;
     // log
     if (err?.response) {
       console.log('openai error: 生成QA错误');
@@ -144,7 +145,6 @@ A2:
     }
 
     // unlock
-    global.qaQueueLen--;
     await TrainingData.findByIdAndUpdate(trainingId, {
       lockTime: new Date('2000/1/1')
     });

src/service/events/generateVector.ts

@@ -3,7 +3,7 @@ import { insertKbItem, PgClient } from '@/service/pg';
 import { openaiEmbedding } from '@/pages/api/openapi/plugin/openaiEmbedding';
 import { TrainingData } from '../models/trainingData';
 import { ERROR_ENUM } from '../errorCode';
-import { TrainingTypeEnum } from '@/constants/plugin';
+import { TrainingModeEnum } from '@/constants/plugin';
 
 /* 索引生成队列。每导入一次，就是一个单独的线程 */
 export async function generateVector(): Promise<any> {
@@ -18,7 +18,7 @@ export async function generateVector(): Promise<any> {
   try {
     const data = await TrainingData.findOneAndUpdate(
       {
-        mode: TrainingTypeEnum.index,
+        mode: TrainingModeEnum.index,
         lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) }
       },
       {
@@ -50,38 +50,6 @@ export async function generateVector(): Promise<any> {
       }
     ];
 
-    // 过滤重复的 qa 内容
-    // const searchRes = await Promise.allSettled(
-    //   dataItems.map(async ({ q, a = '' }) => {
-    //     if (!q) {
-    //       return Promise.reject('q为空');
-    //     }
-
-    //     q = q.replace(/\\n/g, '\n');
-    //     a = a.replace(/\\n/g, '\n');
-
-    //     // Exactly the same data, not push
-    //     try {
-    //       const count = await PgClient.count('modelData', {
-    //         where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['q', q], 'AND', ['a', a]]
-    //       });
-
-    //       if (count > 0) {
-    //         return Promise.reject('已经存在');
-    //       }
-    //     } catch (error) {
-    //       error;
-    //     }
-    //     return Promise.resolve({
-    //       q,
-    //       a
-    //     });
-    //   })
-    // );
-    // const filterData = searchRes
-    //   .filter((item) => item.status === 'fulfilled')
-    //   .map<{ q: string; a: string }>((item: any) => item.value);
-
     // 生成词向量
     const vectors = await openaiEmbedding({
       input: dataItems.map((item) => item.q),
@@ -107,6 +75,7 @@ export async function generateVector(): Promise<any> {
     global.vectorQueueLen--;
     generateVector();
   } catch (err: any) {
+    global.vectorQueueLen--;
     // log
     if (err?.response) {
       console.log('openai error: 生成向量错误');
@@ -125,7 +94,6 @@ export async function generateVector(): Promise<any> {
     }
 
     // unlock
-    global.vectorQueueLen--;
     await TrainingData.findByIdAndUpdate(trainingId, {
       lockTime: new Date('2000/1/1')
     });

src/service/pg.ts

@@ -1,6 +1,5 @@
 import { Pool } from 'pg';
 import type { QueryResultRow } from 'pg';
-import { ModelDataStatusEnum } from '@/constants/model';
 
 export const connectPg = async () => {
   if (global.pgClient) {
@@ -180,8 +179,7 @@ export const insertKbItem = ({
       { key: 'kb_id', value: kbId },
       { key: 'q', value: item.q },
       { key: 'a', value: item.a },
-      { key: 'vector', value: `[${item.vector}]` },
-      { key: 'status', value: ModelDataStatusEnum.ready }
+      { key: 'vector', value: `[${item.vector}]` }
     ])
   });
 };

src/service/response.ts

@@ -45,7 +45,7 @@ export const jsonRes = <T = any>(
     } else if (openaiError[error?.response?.statusText]) {
       msg = openaiError[error.response.statusText];
     }
-    console.log(error?.message || error);
+    console.log(error);
   }
 
   res.json({