v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

projects/app/src/service/events/generateVector.ts

@@ -2,8 +2,10 @@ import { insertData2Dataset } from '@/service/core/dataset/data/controller';
 import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
-import { checkInvalidChunkAndLock, checkTeamAiPointsAndLock } from './utils';
-import { delay } from '@fastgpt/global/common/system/utils';
+import { checkTeamAiPointsAndLock } from './utils';
+import { checkInvalidChunkAndLock } from '@fastgpt/service/core/dataset/training/utils';
+import { addMinutes } from 'date-fns';
+import { addLog } from '@fastgpt/service/common/system/log';
 
 const reduceQueue = () => {
   global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0;
@@ -13,7 +15,8 @@ const reduceQueue = () => {
 
 /* 索引生成队列。每导入一次，就是一个单独的线程 */
 export async function generateVector(): Promise<any> {
-  if (global.vectorQueueLen >= global.systemEnv.vectorMaxProcess) return;
+  const max = global.systemEnv?.vectorMaxProcess || 10;
+  if (global.vectorQueueLen >= max) return;
   global.vectorQueueLen++;
   const start = Date.now();
 
@@ -27,7 +30,7 @@ export async function generateVector(): Promise<any> {
     try {
       const data = await MongoDatasetTraining.findOneAndUpdate(
         {
-          lockTime: { $lte: new Date(Date.now() - 1 * 60 * 1000) },
+          lockTime: { $lte: addMinutes(new Date(), -1) },
           mode: TrainingModeEnum.chunk
         },
         {
@@ -68,7 +71,7 @@ export async function generateVector(): Promise<any> {
         }
       };
     } catch (error) {
-      console.log(`Get Training Data error`, error);
+      addLog.error(`Get Training Data error`, error);
       return {
         error: true
       };
@@ -77,11 +80,12 @@ export async function generateVector(): Promise<any> {
 
   if (done || !data) {
     if (reduceQueue()) {
-      console.log(`【index】Task done`);
+      addLog.info(`[Vector Queue] Done`);
     }
     return;
   }
   if (error) {
+    addLog.error(`[Vector Queue] Error`, { error });
     reduceQueue();
     return generateVector();
   }
@@ -92,6 +96,8 @@ export async function generateVector(): Promise<any> {
     return generateVector();
   }
 
+  addLog.info(`[Vector Queue] Start`);
+
   // create vector and insert
   try {
     // invalid data
@@ -103,7 +109,7 @@ export async function generateVector(): Promise<any> {
     }
 
     // insert to dataset
-    const { charsLength } = await insertData2Dataset({
+    const { tokens } = await insertData2Dataset({
       teamId: data.teamId,
       tmbId: data.tmbId,
       datasetId: data.datasetId,
@@ -119,7 +125,7 @@ export async function generateVector(): Promise<any> {
     pushGenerateVectorUsage({
       teamId: data.teamId,
       tmbId: data.tmbId,
-      charsLength,
+      tokens,
       model: data.model,
       billId: data.billId
     });
@@ -129,7 +135,9 @@ export async function generateVector(): Promise<any> {
     reduceQueue();
     generateVector();
 
-    console.log(`embedding finished, time: ${Date.now() - start}ms`);
+    addLog.info(`[Vector Queue] Finish`, {
+      time: Date.now() - start
+    });
   } catch (err: any) {
     reduceQueue();