v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/service/core/dataset/training/controller.ts

@@ -46,12 +46,16 @@ export async function pushDataListToTrainingQueue({
   } = await getCollectionWithDataset(collectionId);
 
   const checkModelValid = async () => {
-    if (trainingMode === TrainingModeEnum.chunk) {
-      const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
-      if (!vectorModelData) {
-        return Promise.reject(`File model ${vectorModel} is inValid`);
-      }
+    const agentModelData = datasetModelList?.find((item) => item.model === agentModel);
+    if (!agentModelData) {
+      return Promise.reject(`Vector model ${agentModel} is inValid`);
+    }
+    const vectorModelData = vectorModelList?.find((item) => item.model === vectorModel);
+    if (!vectorModelData) {
+      return Promise.reject(`File model ${vectorModel} is inValid`);
+    }
 
+    if (trainingMode === TrainingModeEnum.chunk) {
       return {
         maxToken: vectorModelData.maxToken * 1.3,
         model: vectorModelData.model,
@@ -59,17 +63,14 @@ export async function pushDataListToTrainingQueue({
       };
     }
 
-    if (trainingMode === TrainingModeEnum.qa) {
-      const qaModelData = datasetModelList?.find((item) => item.model === agentModel);
-      if (!qaModelData) {
-        return Promise.reject(`Vector model ${agentModel} is inValid`);
-      }
+    if (trainingMode === TrainingModeEnum.qa || trainingMode === TrainingModeEnum.auto) {
       return {
-        maxToken: qaModelData.maxContext * 0.8,
-        model: qaModelData.model,
+        maxToken: agentModelData.maxContext * 0.8,
+        model: agentModelData.model,
         weight: 0
       };
     }
+
     return Promise.reject(`Training mode "${trainingMode}" is inValid`);
   };
 

packages/service/core/dataset/training/utils.ts

@@ -0,0 +1,41 @@
+import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
+import { addLog } from '../../../common/system/log';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+import { MongoDatasetTraining } from './schema';
+
+export const checkInvalidChunkAndLock = async ({
+  err,
+  errText,
+  data
+}: {
+  err: any;
+  errText: string;
+  data: DatasetTrainingSchemaType;
+}) => {
+  if (err?.response) {
+    addLog.error(`openai error: ${errText}`, {
+      status: err.response?.status,
+      statusText: err.response?.statusText,
+      data: err.response?.data
+    });
+  } else {
+    addLog.error(getErrText(err, errText), err);
+  }
+
+  if (
+    err?.message === 'invalid message format' ||
+    err?.type === 'invalid_request_error' ||
+    err?.code === 500
+  ) {
+    addLog.info('Lock training data');
+    console.log(err);
+
+    try {
+      await MongoDatasetTraining.findByIdAndUpdate(data._id, {
+        lockTime: new Date('2998/5/5')
+      });
+    } catch (error) {}
+    return true;
+  }
+  return false;
+};