Add unit tests for dataset training utilities including MongoDB watch and training queue functionality.

* add node find (#4902)

* add node find

* plugin header

* fix

* fix

* remove

* type

* add searched status

* optimize

* perf: search nodes

---------

Co-authored-by: heheer <heheer@sealos.io>

deploy/docker/docker-compose-milvus.yml

@@ -132,15 +132,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt-mcp-server:
     container_name: fastgpt-mcp-server
-    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
     ports:
       - 3005:3000
     networks:
@@ -150,8 +150,8 @@ services:
       - FASTGPT_ENDPOINT=http://fastgpt:3000
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
     ports:
       - 3000:3000
     networks:

deploy/docker/docker-compose-oceanbase/docker-compose.yml

@@ -109,15 +109,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt-mcp-server:
     container_name: fastgpt-mcp-server
-    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
     ports:
       - 3005:3000
     networks:
@@ -127,8 +127,8 @@ services:
       - FASTGPT_ENDPOINT=http://fastgpt:3000
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
     ports:
       - 3000:3000
     networks:

deploy/docker/docker-compose-pgvector.yml

@@ -96,15 +96,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt-mcp-server:
     container_name: fastgpt-mcp-server
-    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
     ports:
       - 3005:3000
     networks:
@@ -114,8 +114,8 @@ services:
       - FASTGPT_ENDPOINT=http://fastgpt:3000
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
     ports:
       - 3000:3000
     networks:

deploy/docker/docker-compose-zilliz.yml

@@ -72,15 +72,15 @@ services:
 
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt-mcp-server:
     container_name: fastgpt-mcp-server
-    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
     ports:
       - 3005:3000
     networks:
@@ -90,8 +90,8 @@ services:
       - FASTGPT_ENDPOINT=http://fastgpt:3000
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.10 # git
+    image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
     ports:
       - 3000:3000
     networks:

docSite/content/zh-cn/docs/development/upgrading/4910.md

@@ -15,8 +15,8 @@ weight: 790
 
 ### 2. 更新镜像 tag
 
-- 更新 FastGPT 镜像 tag: v4.9.10
+- 更新 FastGPT 镜像 tag: v4.9.10-fix2
-- 更新 FastGPT 商业版镜像 tag: v4.9.10
+- 更新 FastGPT 商业版镜像 tag: v4.9.10-fix2
 - mcp_server 无需更新
 - Sandbox 无需更新
 - AIProxy 无需更新

docSite/content/zh-cn/docs/development/upgrading/4911.md

@@ -10,12 +10,16 @@ weight: 789
 
 ## 🚀 新增内容
 
-1. 工作流中，子流程版本控制，可选择“保持最新版本”，无需手动更新。
+1. 工作流中增加节点搜索功能。
+2. 工作流中，子流程版本控制，可选择“保持最新版本”，无需手动更新。
 
 ## ⚙️ 优化
 
-
+1. 原文缓存改用 gridfs 存储，提高上限。
 
 ## 🐛 修复
 
-1. 工作流中，管理员声明的全局系统工具，无法进行版本管理。
+1. 工作流中，管理员声明的全局系统工具，无法进行版本管理。
+2. 工具调用节点前，有交互节点时，上下文异常。
+3. 修复备份导入，小于 1000 字时，无法分块问题。
+4. 自定义 PDF 解析，无法保存 base64 图片。

packages/global/core/workflow/type/node.d.ts

@@ -125,6 +125,7 @@ export type FlowNodeItemType = FlowNodeTemplateType & {
   nodeId: string;
   parentNodeId?: string;
   isError?: boolean;
+  searchedText?: string;
   debugResult?: {
     status: 'running' | 'success' | 'skipped' | 'failed';
     message?: string;

packages/service/common/buffer/rawText/controller.ts

@@ -0,0 +1,178 @@
+import { retryFn } from '@fastgpt/global/common/system/utils';
+import { connectionMongo } from '../../mongo';
+import { MongoRawTextBufferSchema, bucketName } from './schema';
+import { addLog } from '../../system/log';
+import { setCron } from '../../system/cron';
+import { checkTimerLock } from '../../system/timerLock/utils';
+import { TimerIdEnum } from '../../system/timerLock/constants';
+
+const getGridBucket = () => {
+  return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db!, {
+    bucketName: bucketName
+  });
+};
+
+export const addRawTextBuffer = async ({
+  sourceId,
+  sourceName,
+  text,
+  expiredTime
+}: {
+  sourceId: string;
+  sourceName: string;
+  text: string;
+  expiredTime: Date;
+}) => {
+  const gridBucket = getGridBucket();
+  const metadata = {
+    sourceId,
+    sourceName,
+    expiredTime
+  };
+
+  const buffer = Buffer.from(text);
+
+  const fileSize = buffer.length;
+  // 单块大小：尽可能大，但不超过 14MB，不小于128KB
+  const chunkSizeBytes = (() => {
+    // 计算理想块大小：文件大小 ÷ 目标块数(10)。 并且每个块需要小于 14MB
+    const idealChunkSize = Math.min(Math.ceil(fileSize / 10), 14 * 1024 * 1024);
+
+    // 确保块大小至少为128KB
+    const minChunkSize = 128 * 1024; // 128KB
+
+    // 取理想块大小和最小块大小中的较大值
+    let chunkSize = Math.max(idealChunkSize, minChunkSize);
+
+    // 将块大小向上取整到最接近的64KB的倍数，使其更整齐
+    chunkSize = Math.ceil(chunkSize / (64 * 1024)) * (64 * 1024);
+
+    return chunkSize;
+  })();
+
+  const uploadStream = gridBucket.openUploadStream(sourceId, {
+    metadata,
+    chunkSizeBytes
+  });
+
+  return retryFn(async () => {
+    return new Promise((resolve, reject) => {
+      uploadStream.end(buffer);
+      uploadStream.on('finish', () => {
+        resolve(uploadStream.id);
+      });
+      uploadStream.on('error', (error) => {
+        addLog.error('addRawTextBuffer error', error);
+        resolve('');
+      });
+    });
+  });
+};
+
+export const getRawTextBuffer = async (sourceId: string) => {
+  const gridBucket = getGridBucket();
+
+  return retryFn(async () => {
+    const bufferData = await MongoRawTextBufferSchema.findOne(
+      {
+        'metadata.sourceId': sourceId
+      },
+      '_id metadata'
+    ).lean();
+    if (!bufferData) {
+      return null;
+    }
+
+    // Read file content
+    const downloadStream = gridBucket.openDownloadStream(bufferData._id);
+    const chunks: Buffer[] = [];
+
+    return new Promise<{
+      text: string;
+      sourceName: string;
+    } | null>((resolve, reject) => {
+      downloadStream.on('data', (chunk) => {
+        chunks.push(chunk);
+      });
+
+      downloadStream.on('end', () => {
+        const buffer = Buffer.concat(chunks);
+        const text = buffer.toString('utf8');
+        resolve({
+          text,
+          sourceName: bufferData.metadata?.sourceName || ''
+        });
+      });
+
+      downloadStream.on('error', (error) => {
+        addLog.error('getRawTextBuffer error', error);
+        resolve(null);
+      });
+    });
+  });
+};
+
+export const deleteRawTextBuffer = async (sourceId: string): Promise<boolean> => {
+  const gridBucket = getGridBucket();
+
+  return retryFn(async () => {
+    const buffer = await MongoRawTextBufferSchema.findOne({ 'metadata.sourceId': sourceId });
+    if (!buffer) {
+      return false;
+    }
+
+    await gridBucket.delete(buffer._id);
+    return true;
+  });
+};
+
+export const updateRawTextBufferExpiredTime = async ({
+  sourceId,
+  expiredTime
+}: {
+  sourceId: string;
+  expiredTime: Date;
+}) => {
+  return retryFn(async () => {
+    return MongoRawTextBufferSchema.updateOne(
+      { 'metadata.sourceId': sourceId },
+      { $set: { 'metadata.expiredTime': expiredTime } }
+    );
+  });
+};
+
+export const clearExpiredRawTextBufferCron = async () => {
+  const clearExpiredRawTextBuffer = async () => {
+    addLog.debug('Clear expired raw text buffer start');
+    const gridBucket = getGridBucket();
+
+    return retryFn(async () => {
+      const data = await MongoRawTextBufferSchema.find(
+        {
+          'metadata.expiredTime': { $lt: new Date() }
+        },
+        '_id'
+      ).lean();
+
+      for (const item of data) {
+        await gridBucket.delete(item._id);
+      }
+      addLog.debug('Clear expired raw text buffer end');
+    });
+  };
+
+  setCron('*/10 * * * *', async () => {
+    if (
+      await checkTimerLock({
+        timerId: TimerIdEnum.clearExpiredRawTextBuffer,
+        lockMinuted: 9
+      })
+    ) {
+      try {
+        await clearExpiredRawTextBuffer();
+      } catch (error) {
+        addLog.error('clearExpiredRawTextBufferCron error', error);
+      }
+    }
+  });
+};

packages/service/common/buffer/rawText/schema.ts

@@ -1,33 +1,22 @@
-import { getMongoModel, Schema } from '../../mongo';
+import { getMongoModel, type Types, Schema } from '../../mongo';
-import { type RawTextBufferSchemaType } from './type';
 
-export const collectionName = 'buffer_rawtexts';
+export const bucketName = 'buffer_rawtext';
 
 const RawTextBufferSchema = new Schema({
-  sourceId: {
+  metadata: {
-    type: String,
+    sourceId: { type: String, required: true },
-    required: true
+    sourceName: { type: String, required: true },
-  },
+    expiredTime: { type: Date, required: true }
-  rawText: {
+  }
-    type: String,
-    default: ''
-  },
-  createTime: {
-    type: Date,
-    default: () => new Date()
-  },
-  metadata: Object
 });
+RawTextBufferSchema.index({ 'metadata.sourceId': 'hashed' });
+RawTextBufferSchema.index({ 'metadata.expiredTime': -1 });
 
-try {
+export const MongoRawTextBufferSchema = getMongoModel<{
-  RawTextBufferSchema.index({ sourceId: 1 });
+  _id: Types.ObjectId;
-  //  20 minutes
+  metadata: {
-  RawTextBufferSchema.index({ createTime: 1 }, { expireAfterSeconds: 20 * 60 });
+    sourceId: string;
-} catch (error) {
+    sourceName: string;
-  console.log(error);
+    expiredTime: Date;
-}
+  };
-
+}>(`${bucketName}.files`, RawTextBufferSchema);
-export const MongoRawTextBuffer = getMongoModel<RawTextBufferSchemaType>(
-  collectionName,
-  RawTextBufferSchema
-);

packages/service/common/buffer/rawText/type.d.ts

@@ -1,8 +0,0 @@
-export type RawTextBufferSchemaType = {
-  sourceId: string;
-  rawText: string;
-  createTime: Date;
-  metadata?: {
-    filename: string;
-  };
-};

packages/service/common/file/gridfs/controller.ts

@@ -6,13 +6,13 @@ import { type DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
 import { MongoChatFileSchema, MongoDatasetFileSchema } from './schema';
 import { detectFileEncoding, detectFileEncodingByPath } from '@fastgpt/global/common/file/tools';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
-import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
 import { readRawContentByFileBuffer } from '../read/utils';
 import { gridFsStream2Buffer, stream2Encoding } from './utils';
 import { addLog } from '../../system/log';
-import { readFromSecondary } from '../../mongo/utils';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { Readable } from 'stream';
+import { addRawTextBuffer, getRawTextBuffer } from '../../buffer/rawText/controller';
+import { addMinutes } from 'date-fns';
 
 export function getGFSCollection(bucket: `${BucketNameEnum}`) {
   MongoDatasetFileSchema;
@@ -225,13 +225,11 @@ export const readFileContentFromMongo = async ({
 }> => {
   const bufferId = `${fileId}-${customPdfParse}`;
   // read buffer
-  const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: bufferId }, undefined, {
+  const fileBuffer = await getRawTextBuffer(bufferId);
-    ...readFromSecondary
-  }).lean();
   if (fileBuffer) {
     return {
-      rawText: fileBuffer.rawText,
+      rawText: fileBuffer.text,
-      filename: fileBuffer.metadata?.filename || ''
+      filename: fileBuffer?.sourceName
     };
   }
 
@@ -265,16 +263,13 @@ export const readFileContentFromMongo = async ({
     }
   });
 
-  // < 14M
+  // Add buffer
-  if (fileBuffers.length < 14 * 1024 * 1024 && rawText.trim()) {
+  addRawTextBuffer({
-    MongoRawTextBuffer.create({
+    sourceId: bufferId,
-      sourceId: bufferId,
+    sourceName: file.filename,
-      rawText,
+    text: rawText,
-      metadata: {
+    expiredTime: addMinutes(new Date(), 20)
-        filename: file.filename
+  });
-      }
-    });
-  }
 
   return {
     rawText,

packages/service/common/file/gridfs/schema.ts

@@ -1,16 +1,16 @@
 import { Schema, getMongoModel } from '../../mongo';
 
-const DatasetFileSchema = new Schema({});
+const DatasetFileSchema = new Schema({
-const ChatFileSchema = new Schema({});
+  metadata: Object
+});
+const ChatFileSchema = new Schema({
+  metadata: Object
+});
 
-try {
+DatasetFileSchema.index({ uploadDate: -1 });
-  DatasetFileSchema.index({ uploadDate: -1 });
 
-  ChatFileSchema.index({ uploadDate: -1 });
+ChatFileSchema.index({ uploadDate: -1 });
-  ChatFileSchema.index({ 'metadata.chatId': 1 });
+ChatFileSchema.index({ 'metadata.chatId': 1 });
-} catch (error) {
-  console.log(error);
-}
 
 export const MongoDatasetFileSchema = getMongoModel('dataset.files', DatasetFileSchema);
 export const MongoChatFileSchema = getMongoModel('chat.files', ChatFileSchema);

packages/service/common/file/read/utils.ts

@@ -110,7 +110,7 @@ export const readRawContentByFileBuffer = async ({
 
     return {
       rawText: text,
-      formatText: rawText,
+      formatText: text,
       imageList
     };
   };

packages/service/common/system/timerLock/constants.ts

@@ -5,7 +5,8 @@ export enum TimerIdEnum {
   clearExpiredSubPlan = 'clearExpiredSubPlan',
   updateStandardPlan = 'updateStandardPlan',
   scheduleTriggerApp = 'scheduleTriggerApp',
-  notification = 'notification'
+  notification = 'notification',
+  clearExpiredRawTextBuffer = 'clearExpiredRawTextBuffer'
 }
 
 export enum LockNotificationEnum {

packages/service/core/dataset/collection/controller.ts

@@ -77,7 +77,10 @@ export const createCollectionAndInsertData = async ({
   const chunkSplitter = computeChunkSplitter(createCollectionParams);
   const paragraphChunkDeep = computeParagraphChunkDeep(createCollectionParams);
 
-  if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
+  if (
+    trainingType === DatasetCollectionDataProcessModeEnum.qa ||
+    trainingType === DatasetCollectionDataProcessModeEnum.backup
+  ) {
     delete createCollectionParams.chunkTriggerType;
     delete createCollectionParams.chunkTriggerMinSize;
     delete createCollectionParams.dataEnhanceCollectionName;

packages/service/core/dataset/read.ts

@@ -218,6 +218,10 @@ export const rawText2Chunks = ({
     };
   };
 
+  if (backupParse) {
+    return parseDatasetBackup2Chunks(rawText).chunks;
+  }
+
   // Chunk condition
   // 1. 选择最大值条件，只有超过了最大值(默认为模型的最大值*0.7），才会触发分块
   if (chunkTriggerType === ChunkTriggerConfigTypeEnum.maxSize) {
@@ -240,10 +244,6 @@ export const rawText2Chunks = ({
     }
   }
 
-  if (backupParse) {
-    return parseDatasetBackup2Chunks(rawText).chunks;
-  }
-
   const { chunks } = splitText2Chunks({
     text: rawText,
     chunkSize,

packages/service/core/workflow/dispatch/agent/runTool/index.ts

@@ -86,7 +86,6 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
     });
 
   // Check interactive entry
-  const interactiveResponse = lastInteractive;
   props.node.isEntry = false;
   const hasReadFilesTool = toolNodes.some(
     (item) => item.flowNodeType === FlowNodeTypeEnum.readFiles
@@ -143,7 +142,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
         })
       }
     ];
-    if (interactiveResponse) {
+    if (lastInteractive && isEntry) {
       return value.slice(0, -2);
     }
     return value;
@@ -183,7 +182,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
         toolModel,
         maxRunToolTimes: 30,
         messages: adaptMessages,
-        interactiveEntryToolParams: interactiveResponse?.toolParams
+        interactiveEntryToolParams: lastInteractive?.toolParams
       });
     }
     if (toolModel.functionCall) {
@@ -194,7 +193,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
         toolNodes,
         toolModel,
         messages: adaptMessages,
-        interactiveEntryToolParams: interactiveResponse?.toolParams
+        interactiveEntryToolParams: lastInteractive?.toolParams
       });
     }
 
@@ -224,7 +223,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
       toolNodes,
       toolModel,
       messages: adaptMessages,
-      interactiveEntryToolParams: interactiveResponse?.toolParams
+      interactiveEntryToolParams: lastInteractive?.toolParams
     });
   })();
 

packages/service/core/workflow/dispatch/index.ts

@@ -11,7 +11,6 @@ import type {
   SystemVariablesType
 } from '@fastgpt/global/core/workflow/runtime/type';
 import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type.d';
-import type { FlowNodeOutputItemType } from '@fastgpt/global/core/workflow/type/io.d';
 import type {
   AIChatItemValueItemType,
   ChatHistoryItemResType,

packages/service/core/workflow/dispatch/plugin/run.ts

@@ -17,6 +17,7 @@ import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
 import { getPluginRunUserQuery } from '@fastgpt/global/core/workflow/utils';
 import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
 import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
+import { getUserChatInfoAndAuthTeamPoints } from '../../../../support/permission/auth/team';
 
 type RunPluginProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.forbidStream]?: boolean;
@@ -73,9 +74,11 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
     };
   });
 
+  const { externalProvider } = await getUserChatInfoAndAuthTeamPoints(runningAppInfo.tmbId);
   const runtimeVariables = {
     ...filterSystemVariables(props.variables),
-    appId: String(plugin.id)
+    appId: String(plugin.id),
+    ...(externalProvider ? externalProvider.externalWorkflowVariables : {})
   };
   const { flowResponses, flowUsages, assistantResponses, runTimes } = await dispatchWorkFlow({
     ...props,

packages/service/core/workflow/dispatch/plugin/runApp.ts

@@ -20,6 +20,7 @@ import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { getAppVersionById } from '../../../app/version/controller';
 import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
 import { type ChildrenInteractive } from '@fastgpt/global/core/workflow/template/system/interactive/type';
+import { getUserChatInfoAndAuthTeamPoints } from '../../../../support/permission/auth/team';
 
 type Props = ModuleDispatchProps<{
   [NodeInputKeyEnum.userChatInput]: string;
@@ -97,11 +98,13 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
 
   // Rewrite children app variables
   const systemVariables = filterSystemVariables(variables);
+  const { externalProvider } = await getUserChatInfoAndAuthTeamPoints(appData.tmbId);
   const childrenRunVariables = {
     ...systemVariables,
     ...childrenAppVariables,
     histories: chatHistories,
-    appId: String(appData._id)
+    appId: String(appData._id),
+    ...(externalProvider ? externalProvider.externalWorkflowVariables : {})
   };
 
   const childrenInteractive =

packages/service/core/workflow/dispatch/tools/readFiles.ts

@@ -5,8 +5,6 @@ import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
 import axios from 'axios';
 import { serverRequestBaseUrl } from '../../../../common/api/serverRequest';
-import { MongoRawTextBuffer } from '../../../../common/buffer/rawText/schema';
-import { readFromSecondary } from '../../../../common/mongo/utils';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { detectFileEncoding, parseUrlToFileType } from '@fastgpt/global/common/file/tools';
 import { readRawContentByFileBuffer } from '../../../../common/file/read/utils';
@@ -14,6 +12,8 @@ import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import { type ChatItemType, type UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { addLog } from '../../../../common/system/log';
+import { addRawTextBuffer, getRawTextBuffer } from '../../../../common/buffer/rawText/controller';
+import { addMinutes } from 'date-fns';
 
 type Props = ModuleDispatchProps<{
   [NodeInputKeyEnum.fileUrlList]: string[];
@@ -158,14 +158,12 @@ export const getFileContentFromLinks = async ({
     parseUrlList
       .map(async (url) => {
         // Get from buffer
-        const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: url }, undefined, {
+        const fileBuffer = await getRawTextBuffer(url);
-          ...readFromSecondary
-        }).lean();
         if (fileBuffer) {
           return formatResponseObject({
-            filename: fileBuffer.metadata?.filename || url,
+            filename: fileBuffer.sourceName || url,
             url,
-            content: fileBuffer.rawText
+            content: fileBuffer.text
           });
         }
 
@@ -220,17 +218,12 @@ export const getFileContentFromLinks = async ({
           });
 
           // Add to buffer
-          try {
+          addRawTextBuffer({
-            if (buffer.length < 14 * 1024 * 1024 && rawText.trim()) {
+            sourceId: url,
-              MongoRawTextBuffer.create({
+            sourceName: filename,
-                sourceId: url,
+            text: rawText,
-                rawText,
+            expiredTime: addMinutes(new Date(), 20)
-                metadata: {
+          });
-                  filename: filename
-                }
-              });
-            }
-          } catch (error) {}
 
           return formatResponseObject({ filename, url, content: rawText });
         } catch (error) {

packages/web/components/common/String/HighlightText.tsx

@@ -1,17 +1,26 @@
 import { Box } from '@chakra-ui/react';
-import React from 'react';
+import React, { useMemo } from 'react';
 
 const HighlightText = ({
   rawText,
   matchText,
-  color = 'primary.600'
+  color = 'primary.600',
+  mode = 'text'
 }: {
   rawText: string;
   matchText: string;
   color?: string;
+  mode?: 'text' | 'bg';
 }) => {
-  const regex = new RegExp(`(${matchText})`, 'gi');
+  const { parts } = useMemo(() => {
-  const parts = rawText.split(regex);
+    const regx = new RegExp(`(${matchText})`, 'gi');
+    const parts = rawText.split(regx);
+
+    return {
+      regx,
+      parts
+    };
+  }, [rawText, matchText]);
 
   return (
     <Box>
@@ -28,7 +37,17 @@ const HighlightText = ({
         }
 
         return (
-          <Box as="span" key={index} color={highLight ? color : 'inherit'}>
+          <Box
+            as="span"
+            key={index}
+            {...(mode === 'bg'
+              ? {
+                  bg: highLight ? color : 'transparent'
+                }
+              : {
+                  color: highLight ? color : 'inherit'
+                })}
+          >
             {part}
           </Box>
         );
@@ -37,4 +56,4 @@ const HighlightText = ({
   );
 };
 
-export default HighlightText;
+export default React.memo(HighlightText);

packages/web/hooks/useSystem.ts

@@ -3,6 +3,8 @@ import { useContextSelector } from 'use-context-selector';
 
 export const useSystem = () => {
   const isPc = useContextSelector(useSystemStoreContext, (state) => state.isPc);
+  const isMac =
+    typeof window !== 'undefined' && window.navigator.userAgent.toLocaleLowerCase().includes('mac');
 
-  return { isPc };
+  return { isPc, isMac };
 };

packages/web/i18n/en/workflow.json

@@ -63,6 +63,8 @@
   "field_required": "Required",
   "field_used_as_tool_input": "Used as Tool Call Parameter",
   "filter_description": "Currently supports filtering by tags and creation time. Fill in the format as follows:\n{\n  \"tags\": {\n    \"$and\": [\"Tag 1\",\"Tag 2\"],\n    \"$or\": [\"When there are $and tags, and is effective, or is not effective\"]\n  },\n  \"createTime\": {\n      \"$gte\": \"YYYY-MM-DD HH:mm format, collection creation time greater than this time\",\n      \"$lte\": \"YYYY-MM-DD HH:mm format, collection creation time less than this time, can be used with $gte\"\n  }\n}",
+  "find_tip": "Find node ctrl f",
+  "find_tip_mac": "Find node ⌘ f",
   "foldAll": "Collapse all",
   "form_input_result": "User complete input result",
   "form_input_result_tip": "an object containing the complete result",
@@ -123,18 +125,23 @@
   "max_tokens": "Maximum Tokens",
   "mouse_priority": "Mouse first\n- Press the left button to drag the canvas\n- Hold down shift and left click to select batches",
   "new_context": "New Context",
+  "next": "Next",
+  "no_match_node": "No results",
+  "no_node_found": "No node was not found",
   "not_contains": "Does Not Contain",
   "only_the_reference_type_is_supported": "Only reference type is supported",
   "optional_value_type": "Optional Value Type",
   "optional_value_type_tip": "You can specify one or more data types. When dynamically adding fields, users can only select the configured types.",
   "pan_priority": "Touchpad first\n- Click to batch select\n- Move the canvas with two fingers",
   "pass_returned_object_as_output_to_next_nodes": "Pass the object returned in the code as output to the next nodes. The variable name needs to correspond to the return key.",
+  "please_enter_node_name": "Enter the node name",
   "plugin.Instruction_Tip": "You can configure an instruction to explain the purpose of the plugin. This instruction will be displayed each time the plugin is used. Supports standard Markdown syntax.",
   "plugin.Instructions": "Instructions",
   "plugin.global_file_input": "File links (deprecated)",
   "plugin_file_abandon_tip": "Plugin global file upload has been deprecated, please adjust it as soon as possible. \nRelated functions can be achieved through plug-in input and adding image type input.",
   "plugin_input": "Plugin Input",
   "plugin_output_tool": "When the plug-in is executed as a tool, whether this field responds as a result of the tool",
+  "previous": "Previous",
   "question_classification": "Classify",
   "question_optimization": "Query extension",
   "quote_content_placeholder": "The structure of the reference content can be customized to better suit different scenarios. \nSome variables can be used for template configuration\n\n{{q}} - main content\n\n{{a}} - auxiliary data\n\n{{source}} - source name\n\n{{sourceId}} - source ID\n\n{{index}} - nth reference",
@@ -177,9 +184,9 @@
   "text_content_extraction": "Text Extract",
   "text_to_extract": "Text to Extract",
   "these_variables_will_be_input_parameters_for_code_execution": "These variables will be input parameters for code execution",
-  "tool.tool_result": "Tool operation results",
   "to_add_node": "to add",
   "to_connect_node": "to connect",
+  "tool.tool_result": "Tool operation results",
   "tool_call_termination": "Stop ToolCall",
   "tool_custom_field": "Custom Tool",
   "tool_field": " Tool Field Parameter Configuration",

packages/web/i18n/zh-CN/workflow.json

@@ -63,6 +63,8 @@
   "field_required": "必填",
   "field_used_as_tool_input": "作为工具调用参数",
   "filter_description": "目前支持标签和创建时间过滤，需按照以下格式填写：\n{\n  \"tags\": {\n    \"$and\": [\"标签 1\",\"标签 2\"],\n    \"$or\": [\"有 $and 标签时，and 生效，or 不生效\"]\n  },\n  \"createTime\": {\n      \"$gte\": \"YYYY-MM-DD HH:mm 格式即可，集合的创建时间大于该时间\",\n      \"$lte\": \"YYYY-MM-DD HH:mm 格式即可，集合的创建时间小于该时间,可和 $gte 共同使用\"\n  }\n}",
+  "find_tip": "查找节点 ctrl f",
+  "find_tip_mac": "查找节点  ⌘  f",
   "foldAll": "全部折叠",
   "form_input_result": "用户完整输入结果",
   "form_input_result_tip": "一个包含完整结果的对象",
@@ -123,18 +125,23 @@
   "max_tokens": "最大 Tokens",
   "mouse_priority": "鼠标优先\n- 左键按下后可拖动画布\n- 按住 shift 后左键可批量选择",
   "new_context": "新的上下文",
+  "next": "下一个",
+  "no_match_node": "无结果",
+  "no_node_found": "未搜索到节点",
   "not_contains": "不包含",
   "only_the_reference_type_is_supported": "仅支持引用类型",
   "optional_value_type": "可选的数据类型",
   "optional_value_type_tip": "可以指定 1 个或多个数据类型，用户在动态添加字段时，仅可选择配置的类型",
   "pan_priority": "触摸板优先\n- 单击批量选择\n- 双指移动画布",
   "pass_returned_object_as_output_to_next_nodes": "将代码中 return 的对象作为输出，传递给后续的节点。变量名需要对应 return 的 key",
+  "please_enter_node_name": "请输入节点名称",
   "plugin.Instruction_Tip": "可以配置一段说明，以解释该插件的用途。每次使用插件前，会显示该段说明。支持标准 Markdown 语法。",
   "plugin.Instructions": "使用说明",
   "plugin.global_file_input": "文件链接(弃用)",
   "plugin_file_abandon_tip": "插件全局文件上传已弃用，请尽快调整。可以通过插件输入，添加图片类型输入来实现相关功能。",
   "plugin_input": "插件输入",
   "plugin_output_tool": "插件作为工具执行时，该字段是否作为工具响应结果",
+  "previous": "上一个",
   "question_classification": "问题分类",
   "question_optimization": "问题优化",
   "quote_content_placeholder": "可以自定义引用内容的结构，以更好的适配不同场景。可以使用一些变量来进行模板配置\n{{q}} - 主要内容\n{{a}} - 辅助数据\n{{source}} - 来源名\n{{sourceId}} - 来源ID\n{{index}} - 第 n 个引用",

packages/web/i18n/zh-Hant/workflow.json

@@ -63,6 +63,8 @@
   "field_required": "必填",
   "field_used_as_tool_input": "作為工具呼叫參數",
   "filter_description": "目前支援標籤和建立時間篩選，需按照以下格式填寫：\n{\n  \"tags\": {\n    \"$and\": [\"標籤 1\",\"標籤 2\"],\n    \"$or\": [\"當有 $and 標籤時，$and 才會生效，$or 不會生效\"]\n  },\n  \"createTime\": {\n      \"$gte\": \"YYYY-MM-DD HH:mm 格式，資料集的建立時間大於這個時間\",\n      \"$lte\": \"YYYY-MM-DD HH:mm 格式，資料集的建立時間小於這個時間，可以和 $gte 一起使用\"\n  }\n}",
+  "find_tip": "查找節點 ctrl f",
+  "find_tip_mac": "查找節點 ⌘ f",
   "foldAll": "全部折疊",
   "form_input_result": "使用者完整輸入結果",
   "form_input_result_tip": "一個包含完整結果的物件",
@@ -123,18 +125,23 @@
   "max_tokens": "最大 Token 數",
   "mouse_priority": "滑鼠優先\n- 按下左鍵拖曳畫布\n- 按住 Shift 鍵並點選左鍵可批次選取",
   "new_context": "新的脈絡",
+  "next": "下一個",
+  "no_match_node": "無結果",
+  "no_node_found": "未搜索到節點",
   "not_contains": "不包含",
   "only_the_reference_type_is_supported": "僅支援引用類型",
   "optional_value_type": "可選的資料類型",
   "optional_value_type_tip": "可以指定一或多個資料類型，使用者在動態新增欄位時，只能選擇已設定的類型",
   "pan_priority": "觸控板優先\n- 點選可批次選取\n- 使用兩指移動畫布",
   "pass_returned_object_as_output_to_next_nodes": "將程式碼中 return 的物件作為輸出，傳遞給後續的節點。變數名稱需要對應 return 的鍵值",
+  "please_enter_node_name": "請輸入節點名稱",
   "plugin.Instruction_Tip": "您可以設定一段說明來解釋這個外掛程式的用途。每次使用外掛程式前，都會顯示這段說明。支援標準 Markdown 語法。",
   "plugin.Instructions": "使用說明",
   "plugin.global_file_input": "檔案連結（已淘汰）",
   "plugin_file_abandon_tip": "外掛程式全域檔案上傳功能已淘汰，請儘速調整。您可以透過外掛程式輸入，新增圖片類型輸入來達成相關功能。",
   "plugin_input": "外掛程式輸入",
   "plugin_output_tool": "外掛程式作為工具執行時，這個欄位是否作為工具的回應結果",
+  "previous": "上一個",
   "question_classification": "問題分類",
   "question_optimization": "問題最佳化",
   "quote_content_placeholder": "可以自訂引用內容的結構，以便更好地適應不同場景。可以使用一些變數來設定範本\n{{q}} - 主要內容\n{{a}} - 輔助資料\n{{source}} - 來源名稱\n{{sourceId}} - 來源 ID\n{{index}} - 第 n 個引用",
@@ -177,9 +184,9 @@
   "text_content_extraction": "文字內容擷取",
   "text_to_extract": "要擷取的文字",
   "these_variables_will_be_input_parameters_for_code_execution": "這些變數會作為程式碼執行的輸入參數",
-  "tool.tool_result": "工具運行結果",
   "to_add_node": "添加節點",
   "to_connect_node": "連接節點",
+  "tool.tool_result": "工具運行結果",
   "tool_call_termination": "工具呼叫終止",
   "tool_custom_field": "自訂工具變數",
   "tool_field": "工具參數設定",

projects/app/src/instrumentation.ts

@@ -39,6 +39,12 @@ export async function register() {
       systemStartCb();
       initGlobalVariables();
 
+      try {
+        await preLoadWorker();
+      } catch (error) {
+        console.error('Preload worker error', error);
+      }
+
       // Connect to MongoDB
       await connectMongo(connectionMongo, MONGO_URL);
       connectMongo(connectionLogMongo, MONGO_LOG_URL);
@@ -54,12 +60,6 @@ export async function register() {
       startCron();
       startTrainingQueue(true);
 
-      try {
-        await preLoadWorker();
-      } catch (error) {
-        console.error('Preload worker error', error);
-      }
-
       console.log('Init system success');
     }
   } catch (error) {

projects/app/src/pageComponents/app/detail/Plugin/Header.tsx

@@ -25,16 +25,20 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
 import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
 import { useToast } from '@fastgpt/web/hooks/useToast';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
-import SaveButton from '../Workflow/components/SaveButton';
 import PublishHistories from '../PublishHistoriesSlider';
 import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
 import { WorkflowStatusContext } from '../WorkflowComponents/context/workflowStatusContext';
+import SaveButton from '../Workflow/components/SaveButton';
 
 const Header = () => {
   const { t } = useTranslation();
   const { isPc } = useSystem();
   const router = useRouter();
-  const { toast } = useToast();
+  const { toast: backSaveToast } = useToast({
+    containerStyle: {
+      mt: '60px'
+    }
+  });
 
   const { appDetail, onSaveApp, currentTab } = useContextSelector(AppContext, (v) => v);
   const isV2Workflow = appDetail?.version === 'v2';
@@ -183,6 +187,7 @@ const Header = () => {
                 size={'sm'}
                 leftIcon={<MyIcon name={'core/workflow/debug'} w={['14px', '16px']} />}
                 variant={'whitePrimary'}
+                flexShrink={0}
                 onClick={() => {
                   const data = flowData2StoreDataAndCheck();
                   if (data) {
@@ -211,12 +216,12 @@ const Header = () => {
     onBack,
     onOpenBackConfirm,
     isV2Workflow,
-    showHistoryModal,
     t,
+    showHistoryModal,
     loading,
     onClickSave,
-    flowData2StoreDataAndCheck,
     setShowHistoryModal,
+    flowData2StoreDataAndCheck,
     setWorkflowTestData
   ]);
 
@@ -229,10 +234,11 @@ const Header = () => {
             setShowHistoryModal(false);
           }}
           past={past}
-          onSwitchTmpVersion={onSwitchTmpVersion}
           onSwitchCloudVersion={onSwitchCloudVersion}
+          onSwitchTmpVersion={onSwitchTmpVersion}
         />
       )}
+
       <MyModal
         isOpen={isOpenBackConfirm}
         onClose={onCloseBackConfirm}
@@ -254,7 +260,7 @@ const Header = () => {
                 await onClickSave({});
                 onCloseBackConfirm();
                 onBack();
-                toast({
+                backSaveToast({
                   status: 'success',
                   title: t('app:saved_success'),
                   position: 'top-right'

projects/app/src/pageComponents/app/detail/Workflow/Header.tsx

@@ -13,7 +13,7 @@ import { useTranslation } from 'next-i18next';
 
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useContextSelector } from 'use-context-selector';
-import { WorkflowContext } from '../WorkflowComponents/context';
+import { WorkflowContext, type WorkflowSnapshotsType } from '../WorkflowComponents/context';
 import { AppContext, TabEnum } from '../context';
 import RouteTab from '../RouteTab';
 import { useRouter } from 'next/router';
@@ -25,10 +25,10 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
 import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
 import { useToast } from '@fastgpt/web/hooks/useToast';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
-import SaveButton from './components/SaveButton';
 import PublishHistories from '../PublishHistoriesSlider';
 import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
 import { WorkflowStatusContext } from '../WorkflowComponents/context/workflowStatusContext';
+import SaveButton from '../Workflow/components/SaveButton';
 
 const Header = () => {
   const { t } = useTranslation();
@@ -187,6 +187,7 @@ const Header = () => {
                 size={'sm'}
                 leftIcon={<MyIcon name={'core/workflow/debug'} w={['14px', '16px']} />}
                 variant={'whitePrimary'}
+                flexShrink={0}
                 onClick={() => {
                   const data = flowData2StoreDataAndCheck();
                   if (data) {
@@ -215,12 +216,12 @@ const Header = () => {
     onBack,
     onOpenBackConfirm,
     isV2Workflow,
-    showHistoryModal,
     t,
+    showHistoryModal,
     loading,
     onClickSave,
-    flowData2StoreDataAndCheck,
     setShowHistoryModal,
+    flowData2StoreDataAndCheck,
     setWorkflowTestData
   ]);
 
@@ -228,7 +229,7 @@ const Header = () => {
     <>
       {Render}
       {showHistoryModal && isV2Workflow && currentTab === TabEnum.appEdit && (
-        <PublishHistories
+        <PublishHistories<WorkflowSnapshotsType>
           onClose={() => {
             setShowHistoryModal(false);
           }}

projects/app/src/pageComponents/app/detail/Workflow/components/SaveButton.tsx

@@ -43,6 +43,7 @@ const SaveButton = ({
       Trigger={
         <Button
           size={'sm'}
+          flexShrink={0}
           rightIcon={
             <MyIcon
               name={isSave ? 'core/chat/chevronUp' : 'core/chat/chevronDown'}

projects/app/src/pageComponents/app/detail/Workflow/components/SearchButton.tsx

@@ -0,0 +1,220 @@
+import React, { useState, useCallback } from 'react';
+import { Box, Flex, Button, IconButton, type ButtonProps, Input } from '@chakra-ui/react';
+import { useTranslation } from 'next-i18next';
+import { useContextSelector } from 'use-context-selector';
+import { WorkflowNodeEdgeContext } from '../../WorkflowComponents/context/workflowInitContext';
+import { useReactFlow } from 'reactflow';
+import { useKeyPress, useThrottleEffect } from 'ahooks';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
+import { useSystem } from '@fastgpt/web/hooks/useSystem';
+
+const SearchButton = (props: ButtonProps) => {
+  const { t } = useTranslation();
+  const setNodes = useContextSelector(WorkflowNodeEdgeContext, (state) => state.setNodes);
+  const { fitView } = useReactFlow();
+  const { isMac } = useSystem();
+
+  const [keyword, setKeyword] = useState<string>();
+  const [searchIndex, setSearchIndex] = useState<number>(0);
+  const [searchedNodeCount, setSearchedNodeCount] = useState(0);
+
+  useKeyPress(['ctrl.f', 'meta.f'], (e) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setKeyword('');
+  });
+  useKeyPress(['esc'], (e) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setKeyword(undefined);
+  });
+
+  const onSearch = useCallback(() => {
+    setNodes((nodes) => {
+      if (!keyword) {
+        setSearchIndex(0);
+        setSearchedNodeCount(0);
+        return nodes.map((node) => ({
+          ...node,
+          data: {
+            ...node.data,
+            searchedText: undefined
+          }
+        }));
+      }
+
+      const searchResult = nodes.filter((node) => {
+        const nodeName = t(node.data.name as any);
+        return nodeName.toLowerCase().includes(keyword.toLowerCase());
+      });
+
+      if (searchResult.length === 0) {
+        return nodes.map((node) => ({
+          ...node,
+          data: {
+            ...node.data,
+            searchedText: undefined
+          }
+        }));
+      }
+
+      setSearchedNodeCount(searchResult.length);
+
+      const searchedNode = searchResult[searchIndex] ?? searchResult[0];
+
+      if (searchedNode) {
+        fitView({ nodes: [searchedNode], padding: 0.4 });
+      }
+
+      return nodes.map((node) => ({
+        ...node,
+        selected: node.id === searchedNode.id,
+        data: {
+          ...node.data,
+          searchedText: searchResult.find((item) => item.id === node.id) ? keyword : undefined
+        }
+      }));
+    });
+  }, [keyword, searchIndex]);
+
+  useThrottleEffect(
+    () => {
+      onSearch();
+    },
+    [onSearch],
+    {
+      wait: 500
+    }
+  );
+
+  const goToNextMatch = useCallback(() => {
+    if (searchIndex === searchedNodeCount - 1) {
+      setSearchIndex(0);
+    } else {
+      setSearchIndex(searchIndex + 1);
+    }
+  }, [searchIndex, searchedNodeCount]);
+
+  const goToPreviousMatch = useCallback(() => {
+    if (searchIndex === 0) {
+      setSearchIndex(searchedNodeCount - 1);
+    } else {
+      setSearchIndex(searchIndex - 1);
+    }
+  }, [searchIndex, searchedNodeCount]);
+
+  const clearSearch = useCallback(() => {
+    setKeyword(undefined);
+    setSearchIndex(0);
+    setSearchedNodeCount(0);
+  }, []);
+
+  if (keyword === undefined) {
+    return (
+      <Box position={'absolute'} top={'72px'} left={6} zIndex={1}>
+        <MyTooltip label={isMac ? t('workflow:find_tip_mac') : t('workflow:find_tip')}>
+          <IconButton
+            icon={<MyIcon name="common/searchLight" w="20px" color={'#8A95A7'} />}
+            aria-label=""
+            variant="whitePrimary"
+            size={'mdSquare'}
+            borderRadius={'50%'}
+            bg={'white'}
+            _hover={{ bg: 'white', borderColor: 'primary.300' }}
+            boxShadow={'0px 4px 10px 0px rgba(19, 51, 107, 0.20)'}
+            {...props}
+            onClick={() => setKeyword('')}
+          />
+        </MyTooltip>
+      </Box>
+    );
+  }
+
+  return (
+    <Flex
+      position="absolute"
+      top={3}
+      left="50%"
+      transform="translateX(-50%)"
+      pl={5}
+      pr={4}
+      py={4}
+      zIndex={1}
+      borderRadius={'lg'}
+      bg={'white'}
+      alignItems={'center'}
+      boxShadow={
+        '0px 20px 24px -8px rgba(19, 51, 107, 0.15), 0px 0px 1px 0px rgba(19, 51, 107, 0.15)'
+      }
+      border={'0.5px solid rgba(0, 0, 0, 0.13)'}
+      maxW={['90vw', '550px']}
+      w={'100%'}
+    >
+      <Input
+        flex="1 0 0"
+        h={8}
+        border={'none'}
+        px={0}
+        _focus={{
+          border: 'none',
+          boxShadow: 'none'
+        }}
+        fontSize={'16px'}
+        value={keyword}
+        placeholder={t('workflow:please_enter_node_name')}
+        autoFocus
+        onFocus={onSearch}
+        onChange={(e) => setKeyword(e.target.value)}
+        onKeyDown={(e) => {
+          if (e.key === 'Enter') {
+            e.preventDefault();
+            e.stopPropagation();
+            goToNextMatch();
+          }
+        }}
+      />
+
+      <Box fontSize="sm" color="myGray.600" whiteSpace={'nowrap'} userSelect={'none'}>
+        {searchedNodeCount > 0
+          ? `${searchIndex + 1} / ${searchedNodeCount}`
+          : t('workflow:no_match_node')}
+      </Box>
+
+      {/* Border */}
+      <Box h={5} w={'1px'} bg={'myGray.250'} ml={3} mr={2} />
+
+      <Button
+        size="xs"
+        variant="grayGhost"
+        px={2}
+        isDisabled={searchedNodeCount <= 1}
+        onClick={goToPreviousMatch}
+      >
+        {t('workflow:previous')}
+      </Button>
+      <Button
+        size="xs"
+        variant="grayGhost"
+        px={2}
+        isDisabled={searchedNodeCount <= 1}
+        onClick={goToNextMatch}
+      >
+        {t('workflow:next')}
+      </Button>
+
+      <Flex
+        ml={2}
+        borderRadius="sm"
+        _hover={{ bg: 'myGray.100' }}
+        p={'1'}
+        cursor="pointer"
+        onClick={clearSearch}
+      >
+        <MyIcon name="common/closeLight" w="1.2rem" />
+      </Flex>
+    </Flex>
+  );
+};
+
+export default React.memo(SearchButton);

projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/index.tsx

@@ -1,7 +1,6 @@
 import React from 'react';
 import ReactFlow, { type NodeProps, SelectionMode } from 'reactflow';
 import { Box, IconButton, useDisclosure } from '@chakra-ui/react';
-import { SmallCloseIcon } from '@chakra-ui/icons';
 import { EDGE_TYPE, FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 
 import dynamic from 'next/dynamic';
@@ -20,6 +19,8 @@ import ContextMenu from './components/ContextMenu';
 import { WorkflowNodeEdgeContext, WorkflowInitContext } from '../context/workflowInitContext';
 import { WorkflowEventContext } from '../context/workflowEventContext';
 import NodeTemplatesPopover from './NodeTemplatesPopover';
+import SearchButton from '../../Workflow/components/SearchButton';
+import MyIcon from '@fastgpt/web/components/common/Icon';
 
 const NodeSimple = dynamic(() => import('./nodes/NodeSimple'));
 const nodeTypes: Record<FlowNodeTypeEnum, any> = {
@@ -113,20 +114,22 @@ const Workflow = () => {
         <>
           <IconButton
             position={'absolute'}
-            top={5}
+            top={6}
-            left={5}
+            left={6}
             size={'mdSquare'}
             borderRadius={'50%'}
-            icon={<SmallCloseIcon fontSize={'26px'} />}
+            icon={<MyIcon name="common/addLight" w={'26px'} />}
-            transform={isOpenTemplate ? '' : 'rotate(135deg)'}
             transition={'0.2s ease'}
             aria-label={''}
             zIndex={1}
-            boxShadow={'2px 2px 6px #85b1ff'}
+            boxShadow={
+              '0px 4px 10px 0px rgba(19, 51, 107, 0.20), 0px 0px 1px 0px rgba(19, 51, 107, 0.50)'
+            }
             onClick={() => {
               isOpenTemplate ? onCloseTemplate() : onOpenTemplate();
             }}
           />
+          <SearchButton />
           <NodeTemplatesModal isOpen={isOpenTemplate} onClose={onCloseTemplate} />
           <NodeTemplatesPopover />
         </>

projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/NodeCard.tsx

@@ -36,6 +36,7 @@ import MyTag from '@fastgpt/web/components/common/Tag/index';
 import MySelect from '@fastgpt/web/components/common/MySelect';
 import { useCreation } from 'ahooks';
 import { formatToolError } from '@fastgpt/global/core/app/utils';
+import HighlightText from '@fastgpt/web/components/common/String/HighlightText';
 
 type Props = FlowNodeItemType & {
   children?: React.ReactNode | React.ReactNode[] | string;
@@ -45,6 +46,7 @@ type Props = FlowNodeItemType & {
   w?: string | number;
   h?: string | number;
   selected?: boolean;
+  searchedText?: string;
   menuForbid?: {
     debug?: boolean;
     copy?: boolean;
@@ -70,6 +72,7 @@ const NodeCard = (props: Props) => {
     h = 'full',
     nodeId,
     selected,
+    searchedText,
     menuForbid,
     isTool = false,
     isError = false,
@@ -187,7 +190,12 @@ const NodeCard = (props: Props) => {
                 h={'24px'}
               />
               <Box ml={2} fontSize={'18px'} fontWeight={'medium'} color={'myGray.900'}>
-                {t(name as any)}
+                <HighlightText
+                  rawText={t(name as any)}
+                  matchText={searchedText ?? ''}
+                  mode={'bg'}
+                  color={'#ffe82d'}
+                />
               </Box>
               <Button
                 display={'none'}
@@ -280,6 +288,7 @@ const NodeCard = (props: Props) => {
     nodeId,
     isFolded,
     avatar,
+    searchedText,
     t,
     name,
     showVersion,

projects/app/src/pages/api/core/app/list.ts

@@ -138,18 +138,20 @@ async function handler(req: ApiRequestProps<ListAppBody>): Promise<AppListItemTy
   })();
   const limit = (() => {
     if (getRecentlyChat) return 15;
-    if (searchKey) return 20;
+    if (searchKey) return 50;
-    return 1000;
+    return;
   })();
 
   const myApps = await MongoApp.find(
     findAppsQuery,
-    '_id parentId avatar type name intro tmbId updateTime pluginData inheritPermission'
+    '_id parentId avatar type name intro tmbId updateTime pluginData inheritPermission',
+    {
+      limit: limit
+    }
   )
     .sort({
       updateTime: -1
     })
-    .limit(limit)
     .lean();
 
   // Add app permission and filter apps by read permission

projects/app/src/pages/api/core/dataset/collection/create/fileId.ts

@@ -4,11 +4,11 @@ import { type FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/d
 import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
 import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
-import { MongoRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/schema';
 import { NextAPI } from '@/service/middleware/entry';
 import { type ApiRequestProps } from '@fastgpt/service/type/next';
 import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
 import { type CreateCollectionResponse } from '@/global/core/dataset/api';
+import { deleteRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/controller';
 
 async function handler(
   req: ApiRequestProps<FileIdCreateDatasetCollectionParams>
@@ -52,7 +52,7 @@ async function handler(
   });
 
   // remove buffer
-  await MongoRawTextBuffer.deleteOne({ sourceId: fileId });
+  await deleteRawTextBuffer(fileId);
 
   return {
     collectionId,

projects/app/src/service/common/system/cron.ts

@@ -11,6 +11,7 @@ import { checkTimerLock } from '@fastgpt/service/common/system/timerLock/utils';
 import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants';
 import { addHours } from 'date-fns';
 import { getScheduleTriggerApp } from '@/service/core/app/utils';
+import { clearExpiredRawTextBufferCron } from '@fastgpt/service/common/buffer/rawText/controller';
 
 // Try to run train every minute
 const setTrainingQueueCron = () => {
@@ -83,4 +84,5 @@ export const startCron = () => {
   setClearTmpUploadFilesCron();
   clearInvalidDataCron();
   scheduleTriggerAppCron();
+  clearExpiredRawTextBufferCron();
 };

test/cases/service/core/dataset/training/utils.test.ts

@@ -0,0 +1,128 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import {
+  createDatasetTrainingMongoWatch,
+  startTrainingQueue
+} from '@/service/core/dataset/training/utils';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { generateQA } from '@/service/core/dataset/queues/generateQA';
+import { generateVector } from '@/service/core/dataset/queues/generateVector';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
+
+vi.mock('@/service/core/dataset/queues/generateQA', () => ({
+  generateQA: vi.fn()
+}));
+
+vi.mock('@/service/core/dataset/queues/generateVector', () => ({
+  generateVector: vi.fn()
+}));
+
+vi.mock('@fastgpt/service/core/dataset/training/schema', () => ({
+  MongoDatasetTraining: {
+    watch: vi.fn().mockReturnValue({
+      on: vi.fn()
+    })
+  }
+}));
+
+describe('dataset training utils', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  describe('createDatasetTrainingMongoWatch', () => {
+    it('should setup mongo watch and handle qa mode', () => {
+      const mockOn = vi.fn();
+      vi.mocked(MongoDatasetTraining.watch).mockReturnValue({
+        on: mockOn
+      });
+
+      createDatasetTrainingMongoWatch();
+
+      expect(MongoDatasetTraining.watch).toHaveBeenCalled();
+      expect(mockOn).toHaveBeenCalledWith('change', expect.any(Function));
+
+      // Simulate change event for QA mode
+      const changeHandler = mockOn.mock.calls[0][1];
+      changeHandler({
+        operationType: 'insert',
+        fullDocument: {
+          mode: TrainingModeEnum.qa
+        }
+      });
+
+      expect(generateQA).toHaveBeenCalled();
+      expect(generateVector).not.toHaveBeenCalled();
+    });
+
+    it('should handle chunk mode', () => {
+      const mockOn = vi.fn();
+      vi.mocked(MongoDatasetTraining.watch).mockReturnValue({
+        on: mockOn
+      });
+
+      createDatasetTrainingMongoWatch();
+
+      const changeHandler = mockOn.mock.calls[0][1];
+      changeHandler({
+        operationType: 'insert',
+        fullDocument: {
+          mode: TrainingModeEnum.chunk
+        }
+      });
+
+      expect(generateVector).toHaveBeenCalled();
+      expect(generateQA).not.toHaveBeenCalled();
+    });
+
+    it('should ignore non-insert operations', () => {
+      const mockOn = vi.fn();
+      vi.mocked(MongoDatasetTraining.watch).mockReturnValue({
+        on: mockOn
+      });
+
+      createDatasetTrainingMongoWatch();
+
+      const changeHandler = mockOn.mock.calls[0][1];
+      changeHandler({
+        operationType: 'update',
+        fullDocument: {
+          mode: TrainingModeEnum.qa
+        }
+      });
+
+      expect(generateQA).not.toHaveBeenCalled();
+      expect(generateVector).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('startTrainingQueue', () => {
+    beforeEach(() => {
+      global.systemEnv = {
+        qaMaxProcess: 3
+      };
+    });
+
+    it('should start single process by default', () => {
+      startTrainingQueue();
+
+      expect(generateQA).toHaveBeenCalledTimes(1);
+      expect(generateVector).toHaveBeenCalledTimes(1);
+    });
+
+    it('should start max processes when fast mode enabled', () => {
+      startTrainingQueue(true);
+
+      expect(generateQA).toHaveBeenCalledTimes(3);
+      expect(generateVector).toHaveBeenCalledTimes(3);
+    });
+
+    it('should use default max process when not configured', () => {
+      global.systemEnv = undefined;
+
+      startTrainingQueue(true);
+
+      expect(generateQA).toHaveBeenCalledTimes(10);
+      expect(generateVector).toHaveBeenCalledTimes(10);
+    });
+  });
+});