doc (#4205)

projects/app/src/pages/api/admin/initv491.ts

@@ -7,44 +7,64 @@ import { addLog } from '@fastgpt/service/common/system/log';
 import { delay } from '@fastgpt/global/common/system/utils';
 import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
+import { DatasetDataTextSchemaType } from '@fastgpt/global/core/dataset/type';
+import type { AnyBulkWriteOperation } from '@fastgpt/service/common/mongo';
 
 const updateData = async () => {
   let success = 0;
+
   while (true) {
     try {
-      const data = await MongoDatasetData.find({ initJieba: { $exists: false } }).limit(100);
+      const time = Date.now();
+      const data = await MongoDatasetData.find({
+        initJieba: { $exists: false },
+        updateTime: { $lte: time } // 只需要取旧的数据
+      })
+        .limit(1000)
+        .lean();
       if (data.length === 0) {
         console.log('更新分词完成');
         break;
       }
+      console.log('读取数据完成', Date.now() - time);
+      const dataTextOps: AnyBulkWriteOperation<DatasetDataTextSchemaType>[] = [];
+      const datasetDataIds: string[] = [];
 
-      await Promise.allSettled(
-        data.map(async (item) => {
-          const text = `${item.q} ${item.a}`.trim();
+      // 先进行分词处理
+      for await (const item of data) {
+        const text = `${item.q} ${item.a}`.trim();
+        try {
+          const tokens = await jiebaSplit({ text });
+          dataTextOps.push({
+            updateOne: {
+              filter: { dataId: item._id },
+              update: { $set: { fullTextToken: tokens } }
+            }
+          });
+          datasetDataIds.push(item._id);
+        } catch (error) {
+          console.log(`分词处理错误: ${item._id}`, error);
+        }
+      }
+      console.log('分词处理完成', Date.now() - time);
 
-          try {
-            await mongoSessionRun(async (session) => {
-              await MongoDatasetDataText.updateOne(
-                {
-                  dataId: item._id
-                },
-                {
-                  fullTextToken: await jiebaSplit({ text })
-                },
-                {
-                  session
-                }
-              );
-              // @ts-ignore
-              item.initJieba = true;
-              await item.save({ session });
-            });
-          } catch (error) {
-            console.log(error);
-          }
-        })
-      );
-      success += data.length;
+      await mongoSessionRun(async (session) => {
+        if (dataTextOps.length > 0) {
+          await MongoDatasetDataText.bulkWrite(dataTextOps, { session, ordered: true });
+        }
+        if (datasetDataIds.length > 0) {
+          await MongoDatasetData.updateMany(
+            { _id: { $in: datasetDataIds } },
+            { $set: { initJieba: true } },
+            {
+              session
+            }
+          );
+        }
+      });
+      console.log('保存完成', Date.now() - time);
+
+      success += dataTextOps.length;
       console.log(`成功 ${success}`);
     } catch (error) {
       addLog.error('更新所有旧的 jieba 分词失败', error);