support openGauss vector store (#4819)

deploy/docker/docker-compose-milvus.yml

@@ -132,15 +132,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt-mcp-server:
     container_name: fastgpt-mcp-server
-    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
     ports:
       - 3005:3000
     networks:
@@ -150,8 +150,8 @@ services:
       - FASTGPT_ENDPOINT=http://fastgpt:3000
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
     ports:
       - 3000:3000
     networks:

deploy/docker/docker-compose-oceanbase/docker-compose.yml

@@ -109,15 +109,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt-mcp-server:
     container_name: fastgpt-mcp-server
-    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
     ports:
       - 3005:3000
     networks:
@@ -127,8 +127,8 @@ services:
       - FASTGPT_ENDPOINT=http://fastgpt:3000
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
     ports:
       - 3000:3000
     networks:

deploy/docker/docker-compose-opengauss.yml

@@ -0,0 +1,218 @@
+# 数据库的默认账号和密码仅首次运行时设置有效
+# 如果修改了账号密码，记得改数据库和项目连接参数，别只改一处~
+# 该配置文件只是给快速启动，测试使用。正式使用，记得务必修改账号密码，以及调整合适的知识库参数，共享内存等。
+# 如何无法访问 dockerhub 和 git，可以用阿里云（阿里云没有arm包）
+
+version: '3.3'
+services:
+  # db
+  gs:
+    image: opengauss/opengauss:7.0.0-RC1 # docker hub
+    container_name: gs
+    restart: always
+    # ports: # 生产环境建议不要暴露
+    #   - 5432:5432
+    networks:
+      - fastgpt
+    environment:
+      # 这里的配置只有首次运行生效。修改后，重启镜像是不会生效的。需要把持久化数据删除再重启，才有效果
+      - GS_USER=username
+      - GS_PASSWORD=password
+      - GS_DB=postgres
+    volumes:
+      - ./opengauss/data:/var/lib/opengauss/data
+    healthcheck:
+      test: ['CMD-SHELL', 'netstat -lntp | grep tcp6 > /dev/null 2>&1']
+      interval: 10s
+      timeout: 10s
+      retries: 10
+  mongo:
+    image: mongo:5.0.18 # dockerhub
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/mongo:5.0.18 # 阿里云
+    # image: mongo:4.4.29 # cpu不支持AVX时候使用
+    container_name: mongo
+    restart: always
+    # ports:
+    #   - 27017:27017
+    networks:
+      - fastgpt
+    command: mongod --keyFile /data/mongodb.key --replSet rs0
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=myusername
+      - MONGO_INITDB_ROOT_PASSWORD=mypassword
+    volumes:
+      - ./mongo/data:/data/db
+    entrypoint:
+      - bash
+      - -c
+      - |
+        openssl rand -base64 128 > /data/mongodb.key
+        chmod 400 /data/mongodb.key
+        chown 999:999 /data/mongodb.key
+        echo 'const isInited = rs.status().ok === 1
+        if(!isInited){
+          rs.initiate({
+              _id: "rs0",
+              members: [
+                  { _id: 0, host: "mongo:27017" }
+              ]
+          })
+        }' > /data/initReplicaSet.js
+        # 启动MongoDB服务
+        exec docker-entrypoint.sh "$$@" &
+
+        # 等待MongoDB服务启动
+        until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')"; do
+          echo "Waiting for MongoDB to start..."
+          sleep 2
+        done
+
+        # 执行初始化副本集的脚本
+        mongo -u myusername -p mypassword --authenticationDatabase admin /data/initReplicaSet.js
+
+        # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程
+        wait $$!
+
+  redis:
+    image: redis:7.2-alpine
+    container_name: redis
+    # ports:
+    # - 6379:6379
+    networks:
+      - fastgpt
+    restart: always
+    command: |
+      redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction
+    healthcheck:
+      test: ['CMD', 'redis-cli', '-a', 'mypassword', 'ping']
+      interval: 10s
+      timeout: 3s
+      retries: 3
+      start_period: 30s
+    volumes:
+      - ./redis/data:/data
+
+  # fastgpt
+  sandbox:
+    container_name: sandbox
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.7-fix2 # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.7-fix2 # 阿里云
+    networks:
+      - fastgpt
+    restart: always
+  fastgpt-mcp-server:
+    container_name: fastgpt-mcp-server
+    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.7-fix2 # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.7-fix2 # 阿里云
+    ports:
+      - 3005:3000
+    networks:
+      - fastgpt
+    restart: always
+    environment:
+      - FASTGPT_ENDPOINT=http://fastgpt:3000
+  fastgpt:
+    container_name: fastgpt
+    image: ghcr.io/labring/fastgpt:v4.9.7-fix2 # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.7-fix2 # 阿里云
+    # image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/ghcr.io/labring/fastgpt:v4.8.4-linuxarm64 # openGauss在arm架构上性能更好
+    ports:
+      - 3000:3000
+    networks:
+      - fastgpt
+    depends_on:
+      - mongo
+      - gs
+      - sandbox
+    restart: always
+    environment:
+      # 前端外部可访问的地址，用于自动补全文件资源路径。例如 https:fastgpt.cn，不能填 localhost。这个值可以不填，不填则发给模型的图片会是一个相对路径，而不是全路径，模型可能伪造Host。
+      - FE_DOMAIN=
+      # root 密码，用户名为: root。如果需要修改 root 密码，直接修改这个环境变量，并重启即可。
+      - DEFAULT_ROOT_PSW=1234
+      # AI Proxy 的地址，如果配了该地址，优先使用
+      - AIPROXY_API_ENDPOINT=http://aiproxy:3000
+      # AI Proxy 的 Admin Token，与 AI Proxy 中的环境变量 ADMIN_KEY
+      - AIPROXY_API_TOKEN=aiproxy
+      # 数据库最大连接数
+      - DB_MAX_LINK=30
+      # 登录凭证密钥
+      - TOKEN_KEY=any
+      # root的密钥，常用于升级时候的初始化请求
+      - ROOT_KEY=root_key
+      # 文件阅读加密
+      - FILE_TOKEN_KEY=filetoken
+      # MongoDB 连接参数. 用户名myusername,密码mypassword。
+      - MONGODB_URI=mongodb://myusername:mypassword@mongo:27017/fastgpt?authSource=admin
+      # openGauss 连接参数
+      - OPENGAUSS_URL=opengauss://gaussdb:Huawei12%23%24@gs:9999/test
+      # Redis 连接参数
+      - REDIS_URL=redis://default:mypassword@redis:6379
+      # sandbox 地址
+      - SANDBOX_URL=http://sandbox:3000
+      # 日志等级: debug, info, warn, error
+      - LOG_LEVEL=info
+      - STORE_LOG_LEVEL=warn
+      # 工作流最大运行次数
+      - WORKFLOW_MAX_RUN_TIMES=1000
+      # 批量执行节点，最大输入长度
+      - WORKFLOW_MAX_LOOP_TIMES=100
+      # 自定义跨域，不配置时，默认都允许跨域（多个域名通过逗号分割）
+      - ALLOWED_ORIGINS=
+      # 是否开启IP限制，默认不开启
+      - USE_IP_LIMIT=false
+      # 对话文件过期天数
+      - CHAT_FILE_EXPIRE_TIME=7
+    volumes:
+      - ./config.json:/app/data/config.json
+
+  # AI Proxy
+  aiproxy:
+    image: ghcr.io/labring/aiproxy:v0.1.7
+    # image: registry.cn-hangzhou.aliyuncs.com/labring/aiproxy:v0.1.7 # 阿里云
+    container_name: aiproxy
+    restart: unless-stopped
+    depends_on:
+      aiproxy_pg:
+        condition: service_healthy
+    networks:
+      - fastgpt
+    environment:
+      # 对应 fastgpt 里的AIPROXY_API_TOKEN
+      - ADMIN_KEY=aiproxy
+      # 错误日志详情保存时间（小时）
+      - LOG_DETAIL_STORAGE_HOURS=1
+      # 数据库连接地址
+      - SQL_DSN=postgres://postgres:aiproxy@aiproxy_pg:5432/aiproxy
+      # 最大重试次数
+      - RETRY_TIMES=3
+      # 不需要计费
+      - BILLING_ENABLED=false
+      # 不需要严格检测模型
+      - DISABLE_MODEL_CONFIG=true
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:3000/api/status']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+  aiproxy_pg:
+    image: pgvector/pgvector:0.8.0-pg15 # docker hub
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.8.0-pg15 # 阿里云
+    restart: unless-stopped
+    container_name: aiproxy_pg
+    volumes:
+      - ./aiproxy_pg:/var/lib/postgresql/data
+    networks:
+      - fastgpt
+    environment:
+      TZ: Asia/Shanghai
+      POSTGRES_USER: postgres
+      POSTGRES_DB: aiproxy
+      POSTGRES_PASSWORD: aiproxy
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres', '-d', 'aiproxy']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+networks:
+  fastgpt:

deploy/docker/docker-compose-pgvector.yml

@@ -96,15 +96,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt-mcp-server:
     container_name: fastgpt-mcp-server
-    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
     ports:
       - 3005:3000
     networks:
@@ -114,8 +114,8 @@ services:
       - FASTGPT_ENDPOINT=http://fastgpt:3000
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
     ports:
       - 3000:3000
     networks:

deploy/docker/docker-compose-zilliz.yml

@@ -72,15 +72,15 @@ services:
 
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt-mcp-server:
     container_name: fastgpt-mcp-server
-    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
     ports:
       - 3005:3000
     networks:
@@ -90,8 +90,8 @@ services:
       - FASTGPT_ENDPOINT=http://fastgpt:3000
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
+    image: ghcr.io/labring/fastgpt:v4.9.10 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
     ports:
       - 3000:3000
     networks:

docSite/content/zh-cn/docs/development/upgrading/4910.md

@@ -15,8 +15,8 @@ weight: 790
 
 ### 2. 更新镜像 tag
 
-- 更新 FastGPT 镜像 tag: v4.9.10-fix2
+- 更新 FastGPT 镜像 tag: v4.9.10
-- 更新 FastGPT 商业版镜像 tag: v4.9.10-fix2
+- 更新 FastGPT 商业版镜像 tag: v4.9.10
 - mcp_server 无需更新
 - Sandbox 无需更新
 - AIProxy 无需更新

docSite/content/zh-cn/docs/development/upgrading/4911.md

@@ -14,11 +14,8 @@ weight: 789
 
 ## ⚙️ 优化
 
-1. 原文缓存改用 gridfs 存储，提高上限。
+
 
 ## 🐛 修复
 
-1. 工作流中，管理员声明的全局系统工具，无法进行版本管理。
+1. 工作流中，管理员声明的全局系统工具，无法进行版本管理。
-2. 工具调用节点前，有交互节点时，上下文异常。
-3. 修复备份导入，小于 1000 字时，无法分块问题。
-4. 自定义 PDF 解析，无法保存 base64 图片。

env.d.ts

@@ -15,6 +15,7 @@ declare global {
       MONGODB_LOG_URI?: string;
       PG_URL: string;
       OCEANBASE_URL: string;
+      OPENGAUSS_URL: string;
       MILVUS_ADDRESS: string;
       MILVUS_TOKEN: string;
       SANDBOX_URL: string;

packages/global/core/dataset/api.d.ts

@@ -124,6 +124,13 @@ export type PgSearchRawType = {
   collection_id: string;
   score: number;
 };
+
+export type GsSearchRawType = {
+  id: string;
+  collection_id: string;
+  score: number;
+};
+
 export type PushDatasetDataChunkProps = {
   q: string; // embedding content
   a?: string; // bonus content

packages/service/common/buffer/rawText/controller.ts

@@ -1,179 +0,0 @@
-import { retryFn } from '@fastgpt/global/common/system/utils';
-import { connectionMongo } from '../../mongo';
-import { MongoRawTextBufferSchema, bucketName } from './schema';
-import { addLog } from '../../system/log';
-import { setCron } from '../../system/cron';
-import { checkTimerLock } from '../../system/timerLock/utils';
-import { TimerIdEnum } from '../../system/timerLock/constants';
-
-const getGridBucket = () => {
-  return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db!, {
-    bucketName: bucketName
-  });
-};
-
-export const addRawTextBuffer = async ({
-  sourceId,
-  sourceName,
-  text,
-  expiredTime
-}: {
-  sourceId: string;
-  sourceName: string;
-  text: string;
-  expiredTime: Date;
-}) => {
-  const gridBucket = getGridBucket();
-  const metadata = {
-    sourceId,
-    sourceName,
-    expiredTime
-  };
-
-  const buffer = Buffer.from(text);
-
-  const fileSize = buffer.length;
-  // 单块大小：尽可能大，但不超过 14MB，不小于128KB
-  const chunkSizeBytes = (() => {
-    // 计算理想块大小：文件大小 ÷ 目标块数(10)。 并且每个块需要小于 14MB
-    const idealChunkSize = Math.min(Math.ceil(fileSize / 10), 14 * 1024 * 1024);
-
-    // 确保块大小至少为128KB
-    const minChunkSize = 128 * 1024; // 128KB
-
-    // 取理想块大小和最小块大小中的较大值
-    let chunkSize = Math.max(idealChunkSize, minChunkSize);
-
-    // 将块大小向上取整到最接近的64KB的倍数，使其更整齐
-    chunkSize = Math.ceil(chunkSize / (64 * 1024)) * (64 * 1024);
-
-    return chunkSize;
-  })();
-
-  const uploadStream = gridBucket.openUploadStream(sourceId, {
-    metadata,
-    chunkSizeBytes
-  });
-
-  return retryFn(async () => {
-    return new Promise((resolve, reject) => {
-      uploadStream.end(buffer);
-      uploadStream.on('finish', () => {
-        resolve(uploadStream.id);
-      });
-      uploadStream.on('error', (error) => {
-        addLog.error('addRawTextBuffer error', error);
-        resolve('');
-      });
-    });
-  });
-};
-
-export const getRawTextBuffer = async (sourceId: string) => {
-  const gridBucket = getGridBucket();
-
-  return retryFn(async () => {
-    const bufferData = await MongoRawTextBufferSchema.findOne(
-      {
-        'metadata.sourceId': sourceId
-      },
-      '_id metadata'
-    ).lean();
-    if (!bufferData) {
-      return null;
-    }
-
-    // Read file content
-    const downloadStream = gridBucket.openDownloadStream(bufferData._id);
-    const chunks: Buffer[] = [];
-
-    return new Promise<{
-      text: string;
-      sourceName: string;
-    } | null>((resolve, reject) => {
-      downloadStream.on('data', (chunk) => {
-        chunks.push(chunk);
-      });
-
-      downloadStream.on('end', () => {
-        const buffer = Buffer.concat(chunks);
-        const text = buffer.toString('utf8');
-        resolve({
-          text,
-          sourceName: bufferData.metadata?.sourceName || ''
-        });
-      });
-
-      downloadStream.on('error', (error) => {
-        addLog.error('getRawTextBuffer error', error);
-        resolve(null);
-      });
-    });
-  });
-};
-
-export const deleteRawTextBuffer = async (sourceId: string): Promise<boolean> => {
-  const gridBucket = getGridBucket();
-
-  return retryFn(async () => {
-    const buffer = await MongoRawTextBufferSchema.findOne({ 'metadata.sourceId': sourceId });
-    if (!buffer) {
-      return false;
-    }
-
-    await gridBucket.delete(buffer._id);
-    return true;
-  });
-};
-
-export const updateRawTextBufferExpiredTime = async ({
-  sourceId,
-  expiredTime
-}: {
-  sourceId: string;
-  expiredTime: Date;
-}) => {
-  return retryFn(async () => {
-    return MongoRawTextBufferSchema.updateOne(
-      { 'metadata.sourceId': sourceId },
-      { $set: { 'metadata.expiredTime': expiredTime } }
-    );
-  });
-};
-
-export const clearExpiredRawTextBufferCron = async () => {
-  const clearExpiredRawTextBuffer = async () => {
-    addLog.debug('Clear expired raw text buffer start');
-    const gridBucket = getGridBucket();
-
-    return retryFn(async () => {
-      const data = await MongoRawTextBufferSchema.find(
-        {
-          'metadata.expiredTime': { $lt: new Date() }
-        },
-        '_id'
-      ).lean();
-
-      for (const item of data) {
-        await gridBucket.delete(item._id);
-      }
-      addLog.debug('Clear expired raw text buffer end');
-    });
-  };
-
-  setCron('*/10 * * * *', async () => {
-    if (
-      await checkTimerLock({
-        timerId: TimerIdEnum.clearExpiredRawTextBuffer,
-        lockMinuted: 9
-      })
-    ) {
-      try {
-        await clearExpiredRawTextBuffer();
-      } catch (error) {
-        addLog.error('clearExpiredRawTextBufferCron error', error);
-      }
-    }
-  });
-  clearExpiredRawTextBuffer();
-};

packages/service/common/buffer/rawText/schema.ts

@@ -1,22 +1,33 @@
-import { getMongoModel, type Types, Schema } from '../../mongo';
+import { getMongoModel, Schema } from '../../mongo';
+import { type RawTextBufferSchemaType } from './type';
 
-export const bucketName = 'buffer_rawtext';
+export const collectionName = 'buffer_rawtexts';
 
 const RawTextBufferSchema = new Schema({
-  metadata: {
+  sourceId: {
-    sourceId: { type: String, required: true },
+    type: String,
-    sourceName: { type: String, required: true },
+    required: true
-    expiredTime: { type: Date, required: true }
+  },
-  }
+  rawText: {
+    type: String,
+    default: ''
+  },
+  createTime: {
+    type: Date,
+    default: () => new Date()
+  },
+  metadata: Object
 });
-RawTextBufferSchema.index({ 'metadata.sourceId': 'hashed' });
-RawTextBufferSchema.index({ 'metadata.expiredTime': -1 });
 
-export const MongoRawTextBufferSchema = getMongoModel<{
+try {
-  _id: Types.ObjectId;
+  RawTextBufferSchema.index({ sourceId: 1 });
-  metadata: {
+  //  20 minutes
-    sourceId: string;
+  RawTextBufferSchema.index({ createTime: 1 }, { expireAfterSeconds: 20 * 60 });
-    sourceName: string;
+} catch (error) {
-    expiredTime: Date;
+  console.log(error);
-  };
+}
-}>(`${bucketName}.files`, RawTextBufferSchema);
+
+export const MongoRawTextBuffer = getMongoModel<RawTextBufferSchemaType>(
+  collectionName,
+  RawTextBufferSchema
+);

packages/service/common/buffer/rawText/type.d.ts

@@ -0,0 +1,8 @@
+export type RawTextBufferSchemaType = {
+  sourceId: string;
+  rawText: string;
+  createTime: Date;
+  metadata?: {
+    filename: string;
+  };
+};

packages/service/common/file/gridfs/controller.ts

@@ -6,13 +6,13 @@ import { type DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
 import { MongoChatFileSchema, MongoDatasetFileSchema } from './schema';
 import { detectFileEncoding, detectFileEncodingByPath } from '@fastgpt/global/common/file/tools';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
 import { readRawContentByFileBuffer } from '../read/utils';
 import { gridFsStream2Buffer, stream2Encoding } from './utils';
 import { addLog } from '../../system/log';
+import { readFromSecondary } from '../../mongo/utils';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { Readable } from 'stream';
-import { addRawTextBuffer, getRawTextBuffer } from '../../buffer/rawText/controller';
-import { addMinutes } from 'date-fns';
 
 export function getGFSCollection(bucket: `${BucketNameEnum}`) {
   MongoDatasetFileSchema;
@@ -225,11 +225,13 @@ export const readFileContentFromMongo = async ({
 }> => {
   const bufferId = `${fileId}-${customPdfParse}`;
   // read buffer
-  const fileBuffer = await getRawTextBuffer(bufferId);
+  const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: bufferId }, undefined, {
+    ...readFromSecondary
+  }).lean();
   if (fileBuffer) {
     return {
-      rawText: fileBuffer.text,
+      rawText: fileBuffer.rawText,
-      filename: fileBuffer?.sourceName
+      filename: fileBuffer.metadata?.filename || ''
     };
   }
 
@@ -263,13 +265,16 @@ export const readFileContentFromMongo = async ({
     }
   });
 
-  // Add buffer
+  // < 14M
-  addRawTextBuffer({
+  if (fileBuffers.length < 14 * 1024 * 1024 && rawText.trim()) {
-    sourceId: bufferId,
+    MongoRawTextBuffer.create({
-    sourceName: file.filename,
+      sourceId: bufferId,
-    text: rawText,
+      rawText,
-    expiredTime: addMinutes(new Date(), 20)
+      metadata: {
-  });
+        filename: file.filename
+      }
+    });
+  }
 
   return {
     rawText,

packages/service/common/file/gridfs/schema.ts

@@ -1,16 +1,16 @@
 import { Schema, getMongoModel } from '../../mongo';
 
-const DatasetFileSchema = new Schema({
+const DatasetFileSchema = new Schema({});
-  metadata: Object
+const ChatFileSchema = new Schema({});
-});
-const ChatFileSchema = new Schema({
-  metadata: Object
-});
 
-DatasetFileSchema.index({ uploadDate: -1 });
+try {
+  DatasetFileSchema.index({ uploadDate: -1 });
 
-ChatFileSchema.index({ uploadDate: -1 });
+  ChatFileSchema.index({ uploadDate: -1 });
-ChatFileSchema.index({ 'metadata.chatId': 1 });
+  ChatFileSchema.index({ 'metadata.chatId': 1 });
+} catch (error) {
+  console.log(error);
+}
 
 export const MongoDatasetFileSchema = getMongoModel('dataset.files', DatasetFileSchema);
 export const MongoChatFileSchema = getMongoModel('chat.files', ChatFileSchema);

packages/service/common/file/read/utils.ts

@@ -110,7 +110,7 @@ export const readRawContentByFileBuffer = async ({
 
     return {
       rawText: text,
-      formatText: text,
+      formatText: rawText,
       imageList
     };
   };

packages/service/common/system/timerLock/constants.ts

@@ -5,8 +5,7 @@ export enum TimerIdEnum {
   clearExpiredSubPlan = 'clearExpiredSubPlan',
   updateStandardPlan = 'updateStandardPlan',
   scheduleTriggerApp = 'scheduleTriggerApp',
-  notification = 'notification',
+  notification = 'notification'
-  clearExpiredRawTextBuffer = 'clearExpiredRawTextBuffer'
 }
 
 export enum LockNotificationEnum {

packages/service/common/vectorDB/constants.ts

@@ -3,5 +3,6 @@ export const DatasetVectorTableName = 'modeldata';
 
 export const PG_ADDRESS = process.env.PG_URL;
 export const OCEANBASE_ADDRESS = process.env.OCEANBASE_URL;
+export const OPENGAUSS_ADDRESS = process.env.OPENGAUSS_URL;
 export const MILVUS_ADDRESS = process.env.MILVUS_ADDRESS;
 export const MILVUS_TOKEN = process.env.MILVUS_TOKEN;

packages/service/common/vectorDB/controller.ts

@@ -1,10 +1,11 @@
 /* vector crud */
 import { PgVectorCtrl } from './pg';
 import { ObVectorCtrl } from './oceanbase';
+import { GsVectorCtrl } from './opengauss';
 import { getVectorsByText } from '../../core/ai/embedding';
 import { type DelDatasetVectorCtrlProps, type InsertVectorProps } from './controller.d';
 import { type EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.d';
-import { MILVUS_ADDRESS, PG_ADDRESS, OCEANBASE_ADDRESS } from './constants';
+import { MILVUS_ADDRESS, PG_ADDRESS, OCEANBASE_ADDRESS, OPENGAUSS_ADDRESS } from './constants';
 import { MilvusCtrl } from './milvus';
 import { setRedisCache, getRedisCache, delRedisCache, CacheKeyEnum } from '../redis/cache';
 import { throttle } from 'lodash';
@@ -14,6 +15,7 @@ const getVectorObj = () => {
   if (PG_ADDRESS) return new PgVectorCtrl();
   if (OCEANBASE_ADDRESS) return new ObVectorCtrl();
   if (MILVUS_ADDRESS) return new MilvusCtrl();
+  if (OPENGAUSS_ADDRESS) return new GsVectorCtrl();
 
   return new PgVectorCtrl();
 };

packages/service/common/vectorDB/opengauss/controller.ts

@@ -0,0 +1,188 @@
+import { delay } from '@fastgpt/global/common/system/utils';
+import { addLog } from '../../system/log';
+import { Pool } from 'pg';
+import type { QueryResultRow } from 'pg';
+import { OPENGAUSS_ADDRESS } from '../constants';
+
+export const connectGs = async (): Promise<Pool> => {
+    if (global.gsClient) {
+      return global.gsClient;
+    }
+
+    global.gsClient = new Pool({
+        connectionString: OPENGAUSS_ADDRESS,
+        max: Number(process.env.DB_MAX_LINK || 20),
+        min: 10,
+        keepAlive: true,
+        idleTimeoutMillis: 600000,
+        connectionTimeoutMillis: 20000,
+        query_timeout: 30000,
+        statement_timeout: 40000,
+        idle_in_transaction_session_timeout: 60000
+    });
+
+    global.gsClient.on('error', async (err) => {
+        addLog.error(`openGauss error`, err);
+        global.gsClient?.end();
+        global.gsClient = null;
+    
+        await delay(1000);
+        addLog.info(`Retry connect openGauss`);
+        connectGs();
+      });
+
+    try {
+    await global.gsClient.connect();
+    console.log('openGauss connected');
+    return global.gsClient;
+    } catch (error) {
+    addLog.error(`openGauss connect error`, error);
+    global.gsClient?.end();
+    global.gsClient = null;
+
+    await delay(1000);
+    addLog.info(`Retry connect openGauss`);
+
+    return connectGs();
+    }
+};
+
+type WhereProps = (string | [string, string | number])[];
+type GetProps = {
+  fields?: string[];
+  where?: WhereProps;
+  order?: { field: string; mode: 'DESC' | 'ASC' | string }[];
+  limit?: number;
+  offset?: number;
+};
+
+type DeleteProps = {
+    where: WhereProps;
+  };
+
+type ValuesProps = { key: string; value?: string | number }[];
+type UpdateProps = {
+values: ValuesProps;
+where: WhereProps;
+};
+type InsertProps = {
+values: ValuesProps[];
+};
+
+class GsClass {
+  private getWhereStr(where?: WhereProps) {
+    return where
+      ? `WHERE ${where
+          .map((item) => {
+            if (typeof item === 'string') {
+              return item;
+            }
+            const val = typeof item[1] === 'number' ? item[1] : `'${String(item[1])}'`;
+            return `${item[0]}=${val}`;
+          })
+          .join(' ')}`
+      : '';
+  }
+  private getUpdateValStr(values: ValuesProps) {
+    return values
+      .map((item) => {
+        const val =
+          typeof item.value === 'number'
+            ? item.value
+            : `'${String(item.value).replace(/\'/g, '"')}'`;
+
+        return `${item.key}=${val}`;
+      })
+      .join(',');
+  }
+  private getInsertValStr(values: ValuesProps[]) {
+    return values
+      .map(
+        (items) =>
+          `(${items
+            .map((item) =>
+              typeof item.value === 'number'
+                ? item.value
+                : `'${String(item.value).replace(/\'/g, '"')}'`
+            )
+            .join(',')})`
+      )
+      .join(',');
+  }
+  async select<T extends QueryResultRow = any>(table: string, props: GetProps) {
+    const sql = `SELECT ${
+      !props.fields || props.fields?.length === 0 ? '*' : props.fields?.join(',')
+    }
+      FROM ${table}
+      ${this.getWhereStr(props.where)}
+      ${
+        props.order
+          ? `ORDER BY ${props.order.map((item) => `${item.field} ${item.mode}`).join(',')}`
+          : ''
+      }
+      LIMIT ${props.limit || 10} OFFSET ${props.offset || 0}
+    `;
+
+    const gs = await connectGs();
+    return gs.query<T>(sql);
+  }
+  async count(table: string, props: GetProps) {
+    const sql = `SELECT COUNT(${props?.fields?.[0] || '*'})
+      FROM ${table}
+      ${this.getWhereStr(props.where)}
+    `;
+
+    const gs = await connectGs();
+    return gs.query(sql).then((res) => Number(res.rows[0]?.count || 0));
+  }
+  async delete(table: string, props: DeleteProps) {
+    const sql = `DELETE FROM ${table} ${this.getWhereStr(props.where)}`;
+    const gs = await connectGs();
+    return gs.query(sql);
+  }
+  async update(table: string, props: UpdateProps) {
+    if (props.values.length === 0) {
+      return {
+        rowCount: 0
+      };
+    }
+
+    const sql = `UPDATE ${table} SET ${this.getUpdateValStr(props.values)} ${this.getWhereStr(
+      props.where
+    )}`;
+    const gs = await connectGs();
+    return gs.query(sql);
+  }
+  async insert(table: string, props: InsertProps) {
+    if (props.values.length === 0) {
+      return {
+        rowCount: 0,
+        rows: []
+      };
+    }
+
+    const fields = props.values[0].map((item) => item.key).join(',');
+    const sql = `INSERT INTO ${table} (${fields}) VALUES ${this.getInsertValStr(
+      props.values
+    )} RETURNING id`;
+
+    const gs = await connectGs();
+    return gs.query<{ id: string }>(sql);
+  }
+  async query<T extends QueryResultRow = any>(sql: string) {
+    const gs = await connectGs();
+    const start = Date.now();
+    return gs.query<T>(sql).then((res) => {
+      const time = Date.now() - start;
+
+      if (time > 300) {
+        addLog.warn(`gs query time: ${time}ms, sql: ${sql}`);
+      }
+
+      return res;
+    });
+  }
+}
+
+export const GsClient = new GsClass();
+export const Gs = global.gsClient;

packages/service/common/vectorDB/opengauss/index.ts

@@ -0,0 +1,253 @@
+/* pg vector crud */
+import { DatasetVectorTableName } from '../constants';
+import { delay } from '@fastgpt/global/common/system/utils';
+import { GsClient, connectGs } from './controller';
+import { GsSearchRawType } from '@fastgpt/global/core/dataset/api';
+import type {
+    DelDatasetVectorCtrlProps,
+    EmbeddingRecallCtrlProps,
+    EmbeddingRecallResponse,
+    InsertVectorControllerProps
+  } from '../controller.d';
+  import dayjs from 'dayjs';
+  import { addLog } from '../../system/log';
+
+export class GsVectorCtrl {
+  constructor() {}
+  init = async () => {
+    try {
+      await connectGs();
+      await GsClient.query(`
+        CREATE EXTENSION IF NOT EXISTS vector;
+        CREATE TABLE IF NOT EXISTS ${DatasetVectorTableName} (
+            id BIGSERIAL PRIMARY KEY,
+            vector VECTOR(1536) NOT NULL,
+            team_id VARCHAR(50) NOT NULL,
+            dataset_id VARCHAR(50) NOT NULL,
+            collection_id VARCHAR(50) NOT NULL,
+            createtime TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+        );
+      `);
+
+      await GsClient.query(
+        `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${DatasetVectorTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 128);`
+      );
+      await GsClient.query(
+        `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_collection_index ON ${DatasetVectorTableName} USING btree(team_id, dataset_id, collection_id);`
+      );
+      await GsClient.query(
+        `CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${DatasetVectorTableName} USING btree(createtime);`
+      );
+
+      addLog.info('init pg successful');
+    } catch (error) {
+      addLog.error('init pg error', error);
+    }
+  };
+  insert = async (props: InsertVectorControllerProps): Promise<{ insertId: string }> => {
+    const { teamId, datasetId, collectionId, vector, retry = 3 } = props;
+
+    try {
+      const { rowCount, rows } = await GsClient.insert(DatasetVectorTableName, {
+        values: [
+          [
+            { key: 'vector', value: `[${vector}]` },
+            { key: 'team_id', value: String(teamId) },
+            { key: 'dataset_id', value: String(datasetId) },
+            { key: 'collection_id', value: String(collectionId) }
+          ]
+        ]
+      });
+
+      if (rowCount === 0) {
+        return Promise.reject('insertDatasetData: no insert');
+      }
+
+      return {
+        insertId: rows[0].id
+      };
+    } catch (error) {
+      if (retry <= 0) {
+        return Promise.reject(error);
+      }
+      await delay(500);
+      return this.insert({
+        ...props,
+        retry: retry - 1
+      });
+    }
+  };
+  delete = async (props: DelDatasetVectorCtrlProps): Promise<any> => {
+    const { teamId, retry = 2 } = props;
+
+    const teamIdWhere = `team_id='${String(teamId)}' AND`;
+
+    const where = await (() => {
+      if ('id' in props && props.id) return `${teamIdWhere} id=${props.id}`;
+
+      if ('datasetIds' in props && props.datasetIds) {
+        const datasetIdWhere = `dataset_id IN (${props.datasetIds
+          .map((id) => `'${String(id)}'`)
+          .join(',')})`;
+
+        if ('collectionIds' in props && props.collectionIds) {
+          return `${teamIdWhere} ${datasetIdWhere} AND collection_id IN (${props.collectionIds
+            .map((id) => `'${String(id)}'`)
+            .join(',')})`;
+        }
+
+        return `${teamIdWhere} ${datasetIdWhere}`;
+      }
+
+      if ('idList' in props && Array.isArray(props.idList)) {
+        if (props.idList.length === 0) return;
+        return `${teamIdWhere} id IN (${props.idList.map((id) => String(id)).join(',')})`;
+      }
+      return Promise.reject('deleteDatasetData: no where');
+    })();
+
+    if (!where) return;
+
+    try {
+      await GsClient.delete(DatasetVectorTableName, {
+        where: [where]
+      });
+    } catch (error) {
+      if (retry <= 0) {
+        return Promise.reject(error);
+      }
+      await delay(500);
+      return this.delete({
+        ...props,
+        retry: retry - 1
+      });
+    }
+  };
+  embRecall = async (props: EmbeddingRecallCtrlProps): Promise<EmbeddingRecallResponse> => {
+    const {
+      teamId,
+      datasetIds,
+      vector,
+      limit,
+      forbidCollectionIdList,
+      filterCollectionIdList,
+      retry = 2
+    } = props;
+
+    // Get forbid collection
+    const formatForbidCollectionIdList = (() => {
+      if (!filterCollectionIdList) return forbidCollectionIdList;
+      const list = forbidCollectionIdList
+        .map((id) => String(id))
+        .filter((id) => !filterCollectionIdList.includes(id));
+      return list;
+    })();
+    const forbidCollectionSql =
+      formatForbidCollectionIdList.length > 0
+        ? `AND collection_id NOT IN (${formatForbidCollectionIdList.map((id) => `'${id}'`).join(',')})`
+        : '';
+
+    // Filter by collectionId
+    const formatFilterCollectionId = (() => {
+      if (!filterCollectionIdList) return;
+
+      return filterCollectionIdList
+        .map((id) => String(id))
+        .filter((id) => !forbidCollectionIdList.includes(id));
+    })();
+    const filterCollectionIdSql = formatFilterCollectionId
+      ? `AND collection_id IN (${formatFilterCollectionId.map((id) => `'${id}'`).join(',')})`
+      : '';
+    // Empty data
+    if (formatFilterCollectionId && formatFilterCollectionId.length === 0) {
+      return { results: [] };
+    }
+
+    try {
+      const results: any = await GsClient.query(
+        `BEGIN;
+          SET ob_hnsw_ef_search = ${global.systemEnv?.hnswEfSearch || 100};
+          SELECT id, collection_id, inner_product(vector, [${vector}]) AS score
+            FROM ${DatasetVectorTableName}
+            WHERE team_id='${teamId}'
+              AND dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
+              ${filterCollectionIdSql}
+              ${forbidCollectionSql}
+            ORDER BY score desc APPROXIMATE LIMIT ${limit};
+        COMMIT;`
+      );
+      const rows = results?.[3]?.rows as GsSearchRawType[];
+
+      if (!Array.isArray(rows)) {
+        return {
+          results: []
+        };
+      }
+
+      return {
+        results: rows.map((item) => ({
+          id: String(item.id),
+          collectionId: item.collection_id,
+          score: item.score * -1
+        }))
+      };
+    } catch (error) {
+      if (retry <= 0) {
+        return Promise.reject(error);
+      }
+      return this.embRecall({
+        ...props,
+        retry: retry - 1
+      });
+    }
+  };
+  getVectorDataByTime = async (start: Date, end: Date) => {
+    const { rows } = await GsClient.query<{
+      id: string;
+      team_id: string;
+      dataset_id: string;
+    }>(`SELECT id, team_id, dataset_id
+    FROM ${DatasetVectorTableName}
+    WHERE createtime BETWEEN '${dayjs(start).format('YYYY-MM-DD HH:mm:ss')}' AND '${dayjs(
+      end
+    ).format('YYYY-MM-DD HH:mm:ss')}';
+    `);
+
+    return rows.map((item) => ({
+      id: String(item.id),
+      teamId: item.team_id,
+      datasetId: item.dataset_id
+    }));
+  };
+  getVectorCountByTeamId = async (teamId: string) => {
+    const total = await GsClient.count(DatasetVectorTableName, {
+      where: [['team_id', String(teamId)]]
+    });
+
+    return total;
+  };
+  getVectorCountByDatasetId = async (teamId: string, datasetId: string) => {
+    const total = await GsClient.count(DatasetVectorTableName, {
+      where: [['team_id', String(teamId)], 'and', ['dataset_id', String(datasetId)]]
+    });
+
+    return total;
+  };
+  getVectorCountByCollectionId = async (
+    teamId: string,
+    datasetId: string,
+    collectionId: string
+  ) => {
+    const total = await GsClient.count(DatasetVectorTableName, {
+      where: [
+        ['team_id', String(teamId)],
+        'and',
+        ['dataset_id', String(datasetId)],
+        'and',
+        ['collection_id', String(collectionId)]
+      ]
+    });
+
+    return total;
+  };
+}

packages/service/common/vectorDB/type.d.ts

@@ -6,6 +6,7 @@ declare global {
   var pgClient: Pool | null;
   var obClient: MysqlPool | null;
   var milvusClient: MilvusClient | null;
+  var gsClient: Pool | null;
 }
 
 export type EmbeddingRecallItemType = {

packages/service/core/dataset/collection/controller.ts

@@ -77,10 +77,7 @@ export const createCollectionAndInsertData = async ({
   const chunkSplitter = computeChunkSplitter(createCollectionParams);
   const paragraphChunkDeep = computeParagraphChunkDeep(createCollectionParams);
 
-  if (
+  if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
-    trainingType === DatasetCollectionDataProcessModeEnum.qa ||
-    trainingType === DatasetCollectionDataProcessModeEnum.backup
-  ) {
     delete createCollectionParams.chunkTriggerType;
     delete createCollectionParams.chunkTriggerMinSize;
     delete createCollectionParams.dataEnhanceCollectionName;

packages/service/core/dataset/read.ts

@@ -218,10 +218,6 @@ export const rawText2Chunks = ({
     };
   };
 
-  if (backupParse) {
-    return parseDatasetBackup2Chunks(rawText).chunks;
-  }
-
   // Chunk condition
   // 1. 选择最大值条件，只有超过了最大值(默认为模型的最大值*0.7），才会触发分块
   if (chunkTriggerType === ChunkTriggerConfigTypeEnum.maxSize) {
@@ -244,6 +240,10 @@ export const rawText2Chunks = ({
     }
   }
 
+  if (backupParse) {
+    return parseDatasetBackup2Chunks(rawText).chunks;
+  }
+
   const { chunks } = splitText2Chunks({
     text: rawText,
     chunkSize,

packages/service/core/workflow/dispatch/agent/runTool/index.ts

@@ -86,6 +86,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
     });
 
   // Check interactive entry
+  const interactiveResponse = lastInteractive;
   props.node.isEntry = false;
   const hasReadFilesTool = toolNodes.some(
     (item) => item.flowNodeType === FlowNodeTypeEnum.readFiles
@@ -142,7 +143,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
         })
       }
     ];
-    if (lastInteractive && isEntry) {
+    if (interactiveResponse) {
       return value.slice(0, -2);
     }
     return value;
@@ -182,7 +183,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
         toolModel,
         maxRunToolTimes: 30,
         messages: adaptMessages,
-        interactiveEntryToolParams: lastInteractive?.toolParams
+        interactiveEntryToolParams: interactiveResponse?.toolParams
       });
     }
     if (toolModel.functionCall) {
@@ -193,7 +194,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
         toolNodes,
         toolModel,
         messages: adaptMessages,
-        interactiveEntryToolParams: lastInteractive?.toolParams
+        interactiveEntryToolParams: interactiveResponse?.toolParams
       });
     }
 
@@ -223,7 +224,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
       toolNodes,
       toolModel,
       messages: adaptMessages,
-      interactiveEntryToolParams: lastInteractive?.toolParams
+      interactiveEntryToolParams: interactiveResponse?.toolParams
     });
   })();
 

packages/service/core/workflow/dispatch/tools/readFiles.ts

@@ -5,6 +5,8 @@ import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
 import axios from 'axios';
 import { serverRequestBaseUrl } from '../../../../common/api/serverRequest';
+import { MongoRawTextBuffer } from '../../../../common/buffer/rawText/schema';
+import { readFromSecondary } from '../../../../common/mongo/utils';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { detectFileEncoding, parseUrlToFileType } from '@fastgpt/global/common/file/tools';
 import { readRawContentByFileBuffer } from '../../../../common/file/read/utils';
@@ -12,8 +14,6 @@ import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import { type ChatItemType, type UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { addLog } from '../../../../common/system/log';
-import { addRawTextBuffer, getRawTextBuffer } from '../../../../common/buffer/rawText/controller';
-import { addMinutes } from 'date-fns';
 
 type Props = ModuleDispatchProps<{
   [NodeInputKeyEnum.fileUrlList]: string[];
@@ -158,12 +158,14 @@ export const getFileContentFromLinks = async ({
     parseUrlList
       .map(async (url) => {
         // Get from buffer
-        const fileBuffer = await getRawTextBuffer(url);
+        const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: url }, undefined, {
+          ...readFromSecondary
+        }).lean();
         if (fileBuffer) {
           return formatResponseObject({
-            filename: fileBuffer.sourceName || url,
+            filename: fileBuffer.metadata?.filename || url,
             url,
-            content: fileBuffer.text
+            content: fileBuffer.rawText
           });
         }
 
@@ -218,12 +220,17 @@ export const getFileContentFromLinks = async ({
           });
 
           // Add to buffer
-          addRawTextBuffer({
+          try {
-            sourceId: url,
+            if (buffer.length < 14 * 1024 * 1024 && rawText.trim()) {
-            sourceName: filename,
+              MongoRawTextBuffer.create({
-            text: rawText,
+                sourceId: url,
-            expiredTime: addMinutes(new Date(), 20)
+                rawText,
-          });
+                metadata: {
+                  filename: filename
+                }
+              });
+            }
+          } catch (error) {}
 
           return formatResponseObject({ filename, url, content: rawText });
         } catch (error) {

projects/app/.env.template

@@ -29,6 +29,8 @@ MONGODB_LOG_URI=mongodb://username:password@0.0.0.0:27017/fastgpt?authSource=adm
 PG_URL=postgresql://username:password@host:port/postgres
 # OceanBase 向量库连接参数
 OCEANBASE_URL=
+# openGauss 向量库连接参数
+OPENGAUSS_URL=
 # milvus 向量库连接参数
 MILVUS_ADDRESS=
 MILVUS_TOKEN=

projects/app/src/instrumentation.ts

@@ -39,12 +39,6 @@ export async function register() {
       systemStartCb();
       initGlobalVariables();
 
-      try {
-        await preLoadWorker();
-      } catch (error) {
-        console.error('Preload worker error', error);
-      }
-
       // Connect to MongoDB
       await connectMongo(connectionMongo, MONGO_URL);
       connectMongo(connectionLogMongo, MONGO_LOG_URL);
@@ -60,6 +54,12 @@ export async function register() {
       startCron();
       startTrainingQueue(true);
 
+      try {
+        await preLoadWorker();
+      } catch (error) {
+        console.error('Preload worker error', error);
+      }
+
       console.log('Init system success');
     }
   } catch (error) {

projects/app/src/pages/api/core/app/list.ts

@@ -138,20 +138,18 @@ async function handler(req: ApiRequestProps<ListAppBody>): Promise<AppListItemTy
   })();
   const limit = (() => {
     if (getRecentlyChat) return 15;
-    if (searchKey) return 50;
+    if (searchKey) return 20;
-    return;
+    return 1000;
   })();
 
   const myApps = await MongoApp.find(
     findAppsQuery,
-    '_id parentId avatar type name intro tmbId updateTime pluginData inheritPermission',
+    '_id parentId avatar type name intro tmbId updateTime pluginData inheritPermission'
-    {
-      limit: limit
-    }
   )
     .sort({
       updateTime: -1
     })
+    .limit(limit)
     .lean();
 
   // Add app permission and filter apps by read permission

projects/app/src/pages/api/core/dataset/collection/create/fileId.ts

@@ -4,11 +4,11 @@ import { type FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/d
 import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
 import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { MongoRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/schema';
 import { NextAPI } from '@/service/middleware/entry';
 import { type ApiRequestProps } from '@fastgpt/service/type/next';
 import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
 import { type CreateCollectionResponse } from '@/global/core/dataset/api';
-import { deleteRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/controller';
 
 async function handler(
   req: ApiRequestProps<FileIdCreateDatasetCollectionParams>
@@ -52,7 +52,7 @@ async function handler(
   });
 
   // remove buffer
-  await deleteRawTextBuffer(fileId);
+  await MongoRawTextBuffer.deleteOne({ sourceId: fileId });
 
   return {
     collectionId,

projects/app/src/service/common/system/cron.ts

@@ -11,7 +11,6 @@ import { checkTimerLock } from '@fastgpt/service/common/system/timerLock/utils';
 import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants';
 import { addHours } from 'date-fns';
 import { getScheduleTriggerApp } from '@/service/core/app/utils';
-import { clearExpiredRawTextBufferCron } from '@fastgpt/service/common/buffer/rawText/controller';
 
 // Try to run train every minute
 const setTrainingQueueCron = () => {
@@ -84,5 +83,4 @@ export const startCron = () => {
   setClearTmpUploadFilesCron();
   clearInvalidDataCron();
   scheduleTriggerAppCron();
-  clearExpiredRawTextBufferCron();
 };