V4.9.5 feature (#4520)

* readme

* Add queue log

* Test interactive (#4509)

* Support nested node interaction (#4503)

* feat: Add a new InteractiveContext type and update InteractiveBasicType, adding an optional context property to support more complex interaction state management.

* feat: Enhance workflow interactivity by adding InteractiveContext support and updating dispatch logic to manage nested contexts and entry nodes more effectively.

* feat: Refactor dispatchWorkFlow to utilize InteractiveContext for improved context management

* feat: Enhance entry node resolution by adding validation for entryNodeIds and recursive search in InteractiveContext

* feat: Remove workflowDepth from InteractiveContext and update recovery logic to utilize parentContext for improved context management

* feat: Update getWorkflowEntryNodeIds to use lastInteractive for improved context handling in runtime nodes

* feat: Add lastInteractive support to enhance context management across workflow components

* feat: Enhance interactive workflow by adding stopForInteractive flag and improving memory edge validation in runtime logic

* feat: Refactor InteractiveContext by removing interactiveAppId and updating runtime edge handling in dispatchRunApp for improved context management

* feat: Simplify runtime node and edge initialization in dispatchRunApp by using ternary operators for improved readability and maintainability

* feat: Improve memory edge validation in initWorkflowEdgeStatus by adding detailed comments for better understanding of subset checks and recursive context searching

* feat: Remove commented-out current level information from InteractiveContext for cleaner code and improved readability

* feat: Simplify stopForInteractive check in dispatchWorkFlow for improved code clarity and maintainability

* feat: Remove stopForInteractive handling and related references for improved code clarity and maintainability

* feat: Add interactive response handling in dispatchRunAppNode for enhanced workflow interactivity

* feat: Add context property to InteractiveBasicType and InteractiveNodeType for improved interactivity management

* feat: remove comments

* feat: Remove the node property from ChatDispatchProps to simplify type definitions

* feat: Remove workflowInteractiveResponse from dispatchRunAppNode for cleaner code

* feat: Refactor interactive value handling in chat history processing for improved clarity

* feat: Simplify initWorkflowEdgeStatus logic for better readability and maintainability

* feat: Add workflowInteractiveResponse to dispatchWorkFlow for enhanced functionality

* feat: Enhance interactive response handling with nested children support

* feat: Remove commented-out code for interactive node handling to improve clarity

* feat: remove  InteractiveContext type

* feat: Refactor UserSelectInteractive and UserInputInteractive params for improved structure and clarity

* feat: remove

* feat: The front end supports extracting the deepest interaction parameters to enhance interaction processing

* feat: The front end supports extracting the deepest interaction parameters to enhance interaction processing

* fix: handle undefined interactive values in runtimeEdges and runtimeNodes initialization

* fix: handle undefined interactive values in runtimeNodes and runtimeEdges initialization

* fix: update runtimeNodes and runtimeEdges initialization to use last interactive value

* fix: remove unused imports and replace getLastInteractiveValue with lastInteractive in runtimeEdges initialization

* fix: import WorkflowInteractiveResponseType and handle lastInteractive as undefined in chatTest

* feat: implement extractDeepestInteractive function and refactor usage in AIResponseBox and ChatBox utils

* fix: refactor initWorkflowEdgeStatus and getWorkflowEntryNodeIds calls in dispatchRunAppNode for recovery handling

* fix: ensure lastInteractive is handled consistently as undefined in runtimeEdges and runtimeNodes initialization

* fix: update dispatchFormInput and dispatchUserSelect to use lastInteractive consistently

* fix: update condition checks in dispatchFormInput and dispatchUserSelect to ensure lastInteractive type is validated correctly

* fix: refactor dispatchRunAppNode to replace isRecovery with childrenInteractive for improved clarity in runtimeNodes and runtimeEdges initialization

* refactor: streamline runtimeNodes and runtimeEdges initialization in dispatchRunAppNode for improved readability and maintainability

* fix: update rewriteNodeOutputByHistories function to accept runtimeNodes and interactive as parameters for improved clarity

* fix: simplify interactiveResponse assignment in dispatchWorkFlow for improved clarity

* fix: update entryNodeIds check in getWorkflowEntryNodeIds to ensure it's an array for improved reliability

* remove some invalid code

---------

Co-authored-by: Theresa <63280168+sd0ric4@users.noreply.github.com>

* update doc

* update log

* fix: update debug workflow to conditionally include nextStepSkipNodes… (#4511)

* fix: update debug workflow to conditionally include nextStepSkipNodes based on lastInteractive for improved debugging accuracy

* fix : type error

* remove invalid code

* fix: QA queue

* fix: interactive

* Test log (#4519)

* add log (#4504)

* add log

* update log i18n

* update log

* delete template

* add i18NT

* add team operation log

---------

Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>

* remove search

* update doc

---------

Co-authored-by: Theresa <63280168+sd0ric4@users.noreply.github.com>
Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>

.github/gh-bot.yml

@@ -1,30 +0,0 @@
-version: v1
-debug: true
-action:
-  printConfig: false
-  release:
-    retry: 15s
-    actionName: Release
-    allowOps:
-      - cuisongliu
-bot:
-  prefix: /
-  spe: _
-  allowOps:
-    - sealos-ci-robot
-    - sealos-release-robot
-  email: sealos-ci-robot@sealos.io
-  username: sealos-ci-robot
-repo:
-  org: false
-
-message:
-  success: |
-    🤖 says: Hooray! The action {{.Body}} has been completed successfully. 🎉
-  format_error: |
-    🤖 says: ‼️ There is a formatting issue with the action, kindly verify the action's format.
-  permission_error: |
-    🤖 says: ‼️ The action doesn't have permission to trigger.
-  release_error: |
-    🤖 says: ‼️ Release action failed.
-    Error details: {{.Error}}

.github/workflows/docs-deploy-cf.yml

@@ -1,4 +1,4 @@
-name: Deploy doc image to vercel
+name: Deploy doc image to cf
 
 on:
   workflow_dispatch:
@@ -20,6 +20,11 @@ jobs:
     # The type of runner that the job will run on
     runs-on: ubuntu-22.04
 
+    permissions:
+      contents: write
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+
     # Job outputs
     outputs:
       docs: ${{ steps.filter.outputs.docs }}
@@ -58,20 +63,9 @@ jobs:
       - name: Build
         run: cd docSite && hugo mod get -u github.com/colinwilson/lotusdocs@6d0568e && hugo -v --minify
 
-      # Step 5 - Push our generated site to Vercel
-      - name: Deploy to Vercel
-        uses: amondnet/vercel-action@v25
-        id: vercel-action
-        with:
-          vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
-          vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} #Required
-          vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }} #Required
-          github-comment: false
-          vercel-args: '--prod --local-config ../vercel.json' # Optional
-          working-directory: docSite/public
-
       - name: Deploy to GitHub Pages
-        uses: peaceiris/actions-gh-pages@v3
+        uses: peaceiris/actions-gh-pages@v4
+        if: github.ref == 'refs/heads/main'
         with:
-          github_token: ${{ secrets.GH_PAT }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: docSite/public

.github/workflows/docs-deploy-kubeconfig.yml

@@ -10,6 +10,13 @@ on:
 jobs:
   build-fastgpt-docs-images:
     runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -27,7 +34,6 @@ jobs:
         with:
           # list of Docker images to use as base name for tags
           images: |
-            ${{ secrets.DOCKER_HUB_NAME }}/fastgpt-docs
             ghcr.io/${{ github.repository_owner }}/fastgpt-docs
             registry.cn-hangzhou.aliyuncs.com/${{ secrets.ALI_HUB_USERNAME }}/fastgpt-docs
           tags: |
@@ -40,18 +46,12 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_HUB_NAME }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
       - name: Login to ghcr.io
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
-          username: ${{ github.repository_owner }}
+          username: ${{ github.actor }}
-          password: ${{ secrets.GH_PAT }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Login to Aliyun
         uses: docker/login-action@v3
@@ -70,6 +70,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
     outputs:
       tags: ${{ steps.datetime.outputs.datetime }}
+
   update-docs-image:
     needs: build-fastgpt-docs-images
     runs-on: ubuntu-20.04

.github/workflows/docs-preview.yml

@@ -10,6 +10,12 @@ on:
 jobs:
   # This workflow contains jobs "deploy-production"
   deploy-preview:
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+      pull-requests: write
     # The environment this job references
     environment:
       name: Preview
@@ -32,6 +38,7 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
           submodules: recursive # Fetch submodules
           fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod
+          token: ${{ secrets.GITHUB_TOKEN }}
 
       # Step 2 Detect changes to Docs Content
       - name: Detect changes in doc content
@@ -43,10 +50,6 @@ jobs:
               - 'docSite/content/docs/**'
           base: main
 
-      - name: Add cdn for images
-        run: |
-          sed -i "s#\](/imgs/#\](https://cdn.jsdelivr.net/gh/yangchuansheng/fastgpt-imgs@main/imgs/#g" $(grep -rl "\](/imgs/" docSite/content/zh-cn/docs)
-
       # Step 3 - Install Hugo (specific version)
       - name: Install Hugo
         uses: peaceiris/actions-hugo@v2
@@ -58,39 +61,35 @@ jobs:
       - name: Build
         run: cd docSite && hugo mod get -u github.com/colinwilson/lotusdocs@6d0568e && hugo -v --minify
 
-      # Step 5 - Push our generated site to Vercel
+      # Step 5 - Push our generated site to Cloudflare
-      - name: Deploy to Vercel
+      - name: Deploy to Cloudflare Pages
-        uses: amondnet/vercel-action@v25
+        id: deploy
-        id: vercel-action
+        uses: cloudflare/wrangler-action@v3
         with:
-          vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
+          apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }}
-          vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} #Required
+          accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
-          vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }} #Required
+          command: pages deploy ./docSite/public --project-name=fastgpt-doc
-          github-comment: false
+          packageManager: npm
-          vercel-args: '--local-config ../vercel.json' # Optional
+
-          working-directory: docSite/public
+      - name: Create deployment status comment
-          alias-domains: | #Optional
+        if: always()
-            fastgpt-staging.vercel.app
-  docsOutput:
-    needs: [deploy-preview]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.ref }}
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-      - name: Write md
-        run: |
-          echo "# 🤖 Generated by deploy action" > report.md
-          echo "[👀 Visit Preview](${{ needs.deploy-preview.outputs.url }})" >> report.md
-          cat report.md
-      - name: Gh Rebot for Sealos
-        uses: labring/gh-rebot@v0.0.6
-        if: ${{ (github.event_name == 'pull_request_target') }}
-        with:
-          version: v0.0.6
         env:
-          GH_TOKEN: '${{ secrets.GH_PAT }}'
+          JOB_STATUS: ${{ job.status }}
-          SEALOS_TYPE: 'pr_comment'
+          PREVIEW_URL: ${{ steps.deploy.outputs.deployment-url }}
-          SEALOS_FILENAME: 'report.md'
+        uses: actions/github-script@v6
-          SEALOS_REPLACE_TAG: 'DEFAULT_REPLACE_DEPLOY'
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const success = process.env.JOB_STATUS === 'success';
+            const deploymentUrl = `${process.env.PREVIEW_URL}`;
+            const status = success ? '✅ Success' : '❌ Failed';
+            console.log(process.env.JOB_STATUS);
+
+            const commentBody = `**Deployment Status: ${status}**
+            ${success ? `🔗 Preview URL: ${deploymentUrl}` : ''}`;
+
+            await github.rest.issues.createComment({
+              ...context.repo,
+              issue_number: context.payload.pull_request.number,
+              body: commentBody
+            });

.github/workflows/docs-sync_imgs.yml

@@ -1,6 +1,6 @@
 name: Sync images
 on:
-  pull_request_target:
+  pull_request:
     branches:
       - main
     paths:
@@ -15,13 +15,6 @@ jobs:
   sync:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-        if: ${{ (github.event_name == 'pull_request_target') }}
-        with:
-          ref: ${{ github.event.pull_request.head.ref }}
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-
       - name: Checkout
         uses: actions/checkout@v3
 

.github/workflows/fastgpt-build-image-personal.yml

@@ -9,6 +9,11 @@ on:
       - 'main'
 jobs:
   build-fastgpt-images:
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
     runs-on: ubuntu-20.04
     if: github.repository != 'labring/FastGPT'
     steps:
@@ -32,7 +37,7 @@ jobs:
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GH_PAT }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Set DOCKER_REPO_TAGGED based on branch or tag
         run: |
           echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt:latest" >> $GITHUB_ENV

.github/workflows/fastgpt-build-image.yml

@@ -9,6 +9,11 @@ on:
       - 'v*'
 jobs:
   build-fastgpt-images:
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
     runs-on: ubuntu-20.04
     steps:
       # install env
@@ -39,7 +44,7 @@ jobs:
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GH_PAT }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Login to Ali Hub
         uses: docker/login-action@v2
         with:
@@ -91,6 +96,11 @@ jobs:
           -t ${Docker_Hub_Latest} \
           .
   build-fastgpt-images-sub-route:
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
     runs-on: ubuntu-20.04
     steps:
       # install env
@@ -121,7 +131,7 @@ jobs:
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GH_PAT }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Login to Ali Hub
         uses: docker/login-action@v2
         with:
@@ -174,6 +184,11 @@ jobs:
           -t ${Docker_Hub_Latest} \
           .
   build-fastgpt-images-sub-route-gchat:
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
     runs-on: ubuntu-20.04
     steps:
       # install env
@@ -204,7 +219,7 @@ jobs:
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GH_PAT }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Login to Ali Hub
         uses: docker/login-action@v2
         with:

.github/workflows/fastgpt-preview-image.yml

@@ -5,6 +5,13 @@ on:
 
 jobs:
   preview-fastgpt-images:
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+      pull-requests: write
+
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -12,8 +19,9 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
-          submodules: recursive # Fetch submodules
           fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod
+          token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:
@@ -25,15 +33,18 @@ jobs:
           key: ${{ runner.os }}-buildx-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-buildx-
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GH_PAT }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Set DOCKER_REPO_TAGGED based on branch or tag
         run: |
           echo "DOCKER_REPO_TAGGED=ghcr.io/${{ github.repository_owner }}/fastgpt-pr:${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
+
       - name: Build image for PR
         env:
           DOCKER_REPO_TAGGED: ${{ env.DOCKER_REPO_TAGGED }}
@@ -48,20 +59,13 @@ jobs:
           --cache-to=type=local,dest=/tmp/.buildx-cache \
           -t ${DOCKER_REPO_TAGGED} \
           .
-      # Add write md step after build
+      - uses: actions/github-script@v7
-      - name: Write md
-        run: |
-          echo "# 🤖 Generated by deploy action" > report.md
-          echo "📦 Preview Image: \`${DOCKER_REPO_TAGGED}\`" >> report.md
-          cat report.md
-
-      - name: Gh Rebot for Sealos
-        uses: labring/gh-rebot@v0.0.6
-        if: ${{ (github.event_name == 'pull_request_target') }}
         with:
-          version: v0.0.6
+          github-token: ${{secrets.GITHUB_TOKEN}}
-        env:
+          script: |
-          GH_TOKEN: '${{ secrets.GH_PAT }}'
+            github.rest.issues.createComment({
-          SEALOS_TYPE: 'pr_comment'
+              issue_number: context.issue.number,
-          SEALOS_FILENAME: 'report.md'
+              owner: context.repo.owner,
-          SEALOS_REPLACE_TAG: 'DEFAULT_REPLACE_DEPLOY'
+              repo: context.repo.repo,
+              body: 'Preview Image: `${{ env.DOCKER_REPO_TAGGED }}`'
+            })

.github/workflows/fastgpt-test.yaml

@@ -15,6 +15,9 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
       - uses: pnpm/action-setup@v4
         with:
           version: 10

.github/workflows/helm-release.yaml

@@ -8,6 +8,11 @@ on:
 
 jobs:
   helm:
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout
@@ -20,7 +25,7 @@ jobs:
         run: echo "tag=$(git describe --tags)" >> $GITHUB_OUTPUT
       - name: Release Helm
         run: |
-          echo ${{ secrets.GH_PAT }} | helm registry login ghcr.io -u ${{ github.repository_owner }} --password-stdin
+          echo ${{ secrets.GITHUB_TOKEN }} | helm registry login ghcr.io -u ${{ github.repository_owner }} --password-stdin
           export APP_VERSION=${{ steps.vars.outputs.tag }}
           export HELM_VERSION=${{ steps.vars.outputs.tag }}
           export HELM_REPO=ghcr.io/${{ github.repository_owner }}

.github/workflows/sandbox-build-image.yml

@@ -8,6 +8,11 @@ on:
       - 'v*'
 jobs:
   build-fastgpt-sandbox-images:
+    permissions:
+      packages: write
+      contents: read
+      attestations: write
+      id-token: write
     runs-on: ubuntu-20.04
     steps:
       # install env
@@ -38,7 +43,7 @@ jobs:
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
-          password: ${{ secrets.GH_PAT }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Login to Ali Hub
         uses: docker/login-action@v2
         with:

README.md

@@ -10,7 +10,7 @@
   <a href="./README_ja.md">日语</a>
 </p>
 
-FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开箱即用的数据处理、模型调用等能力。同时可以通过 Flow 可视化进行工作流编排，从而实现复杂的问答场景！
+FastGPT 是一个 AI Agent 构建平台，提供开箱即用的数据处理、模型调用等能力，同时可以通过 Flow 可视化进行工作流编排，从而实现复杂的应用场景！
 
 </div>
 

deploy/docker/docker-compose-milvus.yml

@@ -110,19 +110,31 @@ services:
 
         # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程
         wait $$!
+  redis:
+    image: redis:7.2-alpine
+    container_name: redis
+    # ports:
+    # - 6379:6379
+    networks:
+      - fastgpt
+    restart: always
+    command: |
+      redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction
+    volumes:
+      - ./redis/data:/data
 
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.1-fix2 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.4 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.4 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.1-fix2 # git
+    image: ghcr.io/labring/fastgpt:v4.9.4 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.4 # 阿里云
     ports:
       - 3000:3000
     networks:
@@ -157,6 +169,8 @@ services:
       # zilliz 连接参数
       - MILVUS_ADDRESS=http://milvusStandalone:19530
       - MILVUS_TOKEN=none
+      # Redis 地址
+      - REDIS_URL=redis://default:mypassword@redis:6379
       # sandbox 地址
       - SANDBOX_URL=http://sandbox:3000
       # 日志等级: debug, info, warn, error
@@ -170,6 +184,8 @@ services:
       - ALLOWED_ORIGINS=
       # 是否开启IP限制，默认不开启
       - USE_IP_LIMIT=false
+      # 对话文件过期天数
+      - CHAT_FILE_EXPIRE_TIME=7
     volumes:
       - ./config.json:/app/data/config.json
 

deploy/docker/docker-compose-oceanbase/docker-compose.yml

@@ -0,0 +1,202 @@
+# 数据库的默认账号和密码仅首次运行时设置有效
+# 如果修改了账号密码，记得改数据库和项目连接参数，别只改一处~
+# 该配置文件只是给快速启动，测试使用。正式使用，记得务必修改账号密码，以及调整合适的知识库参数，共享内存等。
+# 如何无法访问 dockerhub 和 git，可以用阿里云（阿里云没有arm包）
+
+version: '3.3'
+services:
+  # vector db
+  ob:
+    image: oceanbase/oceanbase-ce # docker hub
+    # image: quay.io/oceanbase/oceanbase-ce:4.3.5.1-101000042025031818 # 镜像
+    container_name: ob
+    restart: always
+    # ports: # 生产环境建议不要暴露
+    #   - 2881:2881
+    networks:
+      - fastgpt
+    environment:
+      # 这里的配置只有首次运行生效。修改后，重启镜像是不会生效的。需要把持久化数据删除再重启，才有效果
+      - OB_SYS_PASSWORD=obsyspassword
+      # 不同于传统数据库，OceanBase 数据库的账号包含更多字段，包括用户名、租户名和集群名。经典格式为“用户名@租户名#集群名”
+      # 比如用mysql客户端连接时，根据本文件的默认配置，应该指定 “-uroot@tenantname”
+      - OB_TENANT_NAME=tenantname
+      - OB_TENANT_PASSWORD=tenantpassword
+      # MODE分为MINI和NORMAL， 后者会最大程度使用主机资源
+      - MODE=NORMAL
+      - OB_SERVER_IP=127.0.0.1
+      # 更多环境变量配置见oceanbase官方文档： https://www.oceanbase.com/docs/common-oceanbase-database-cn-1000000002013494
+    volumes:
+      - ./ob/data:/root/ob
+      - ./ob/config:/root/.obd/cluster
+      - ./init.sql:/root/boot/init.d/init.sql
+    healthcheck:
+    # obclient -h127.0.0.1 -P2881 -uroot@tenantname -ptenantpassword -e "SELECT 1;"
+      test: ["CMD-SHELL", "obclient -h$OB_SERVER_IP -P2881 -uroot@$OB_TENANT_NAME -p$OB_TENANT_PASSWORD -e \"SELECT 1;\""]
+      interval: 30s
+      timeout: 10s
+      retries: 1000
+      start_period: 10s
+  mongo:
+    image: mongo:5.0.18 # dockerhub
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/mongo:5.0.18 # 阿里云
+    # image: mongo:4.4.29 # cpu不支持AVX时候使用
+    container_name: mongo
+    restart: always
+    # ports:
+    #   - 27017:27017
+    networks:
+      - fastgpt
+    command: mongod --keyFile /data/mongodb.key --replSet rs0
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=myusername
+      - MONGO_INITDB_ROOT_PASSWORD=mypassword
+    volumes:
+      - ./mongo/data:/data/db
+    entrypoint:
+      - bash
+      - -c
+      - |
+        openssl rand -base64 128 > /data/mongodb.key
+        chmod 400 /data/mongodb.key
+        chown 999:999 /data/mongodb.key
+        echo 'const isInited = rs.status().ok === 1
+        if(!isInited){
+          rs.initiate({
+              _id: "rs0",
+              members: [
+                  { _id: 0, host: "mongo:27017" }
+              ]
+          })
+        }' > /data/initReplicaSet.js
+        # 启动MongoDB服务
+        exec docker-entrypoint.sh "$$@" &
+
+        # 等待MongoDB服务启动
+        until mongo -u myusername -p mypassword --authenticationDatabase admin --eval "print('waited for connection')"; do
+          echo "Waiting for MongoDB to start..."
+          sleep 2
+        done
+
+        # 执行初始化副本集的脚本
+        mongo -u myusername -p mypassword --authenticationDatabase admin /data/initReplicaSet.js
+
+        # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程
+        wait $$!
+
+  # fastgpt
+  sandbox:
+    container_name: sandbox
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.3 # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.3 # 阿里云
+    networks:
+      - fastgpt
+    restart: always
+  fastgpt:
+    container_name: fastgpt
+    image: ghcr.io/labring/fastgpt:v4.9.3 # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.3 # 阿里云
+    ports:
+      - 3000:3000
+    networks:
+      - fastgpt
+    depends_on:
+      mongo:
+        condition: service_started
+      ob:
+        condition: service_healthy
+      sandbox:
+        condition: service_started
+    restart: always
+    environment:
+      # 前端外部可访问的地址，用于自动补全文件资源路径。例如 https:fastgpt.cn，不能填 localhost。这个值可以不填，不填则发给模型的图片会是一个相对路径，而不是全路径，模型可能伪造Host。
+      - FE_DOMAIN=
+      # root 密码，用户名为: root。如果需要修改 root 密码，直接修改这个环境变量，并重启即可。
+      - DEFAULT_ROOT_PSW=1234
+      # # AI Proxy 的地址，如果配了该地址，优先使用
+      # - AIPROXY_API_ENDPOINT=http://aiproxy:3000
+      # # AI Proxy 的 Admin Token，与 AI Proxy 中的环境变量 ADMIN_KEY
+      # - AIPROXY_API_TOKEN=aiproxy
+      # 模型中转地址（如果用了 AI Proxy，下面 2 个就不需要了，旧版 OneAPI 用户，使用下面的变量）
+      - # openai 基本地址，可用作中转。
+      - OPENAI_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1
+      - # OpenAI API Key
+      - CHAT_API_KEY=sk-8990fa15a34b464a805237cfe9561f11
+      # 数据库最大连接数
+      - DB_MAX_LINK=30
+      # 登录凭证密钥
+      - TOKEN_KEY=any
+      # root的密钥，常用于升级时候的初始化请求
+      - ROOT_KEY=root_key
+      # 文件阅读加密
+      - FILE_TOKEN_KEY=filetoken
+      # MongoDB 连接参数. 用户名myusername,密码mypassword。
+      - MONGODB_URI=mongodb://myusername:mypassword@mongo:27017/fastgpt?authSource=admin
+      # OceanBase 向量库连接参数
+      - OCEANBASE_URL=mysql://root%40tenantname:tenantpassword@ob:2881/test
+      # sandbox 地址
+      - SANDBOX_URL=http://sandbox:3000
+      # 日志等级: debug, info, warn, error
+      - LOG_LEVEL=info
+      - STORE_LOG_LEVEL=warn
+      # 工作流最大运行次数
+      - WORKFLOW_MAX_RUN_TIMES=1000
+      # 批量执行节点，最大输入长度
+      - WORKFLOW_MAX_LOOP_TIMES=100
+      # 自定义跨域，不配置时，默认都允许跨域（多个域名通过逗号分割）
+      - ALLOWED_ORIGINS=
+      # 是否开启IP限制，默认不开启
+      - USE_IP_LIMIT=false
+    volumes:
+      - ./config.json:/app/data/config.json
+
+  # AI Proxy
+  aiproxy:
+    image: ghcr.io/labring/aiproxy:v0.1.5
+    # image: registry.cn-hangzhou.aliyuncs.com/labring/aiproxy:v0.1.3 # 阿里云
+    container_name: aiproxy
+    restart: unless-stopped
+    depends_on:
+      aiproxy_pg:
+        condition: service_healthy
+    networks:
+      - fastgpt
+    environment:
+      # 对应 fastgpt 里的AIPROXY_API_TOKEN
+      - ADMIN_KEY=aiproxy
+      # 错误日志详情保存时间（小时）
+      - LOG_DETAIL_STORAGE_HOURS=1
+      # 数据库连接地址
+      - SQL_DSN=postgres://postgres:aiproxy@aiproxy_pg:5432/aiproxy
+      # 最大重试次数
+      - RETRY_TIMES=3
+      # 不需要计费
+      - BILLING_ENABLED=false
+      # 不需要严格检测模型
+      - DISABLE_MODEL_CONFIG=true
+    healthcheck:
+      test: ['CMD', 'curl', '-f', 'http://localhost:3000/api/status']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+  aiproxy_pg:
+    image: pgvector/pgvector:0.8.0-pg15 # docker hub
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.8.0-pg15 # 阿里云
+    restart: unless-stopped
+    container_name: aiproxy_pg
+    volumes:
+      - ./aiproxy_pg:/var/lib/postgresql/data
+    networks:
+      - fastgpt
+    environment:
+      TZ: Asia/Shanghai
+      POSTGRES_USER: postgres
+      POSTGRES_DB: aiproxy
+      POSTGRES_PASSWORD: aiproxy
+    healthcheck:
+      test: ['CMD', 'pg_isready', '-U', 'postgres', '-d', 'aiproxy']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+networks:
+  fastgpt:

deploy/docker/docker-compose-oceanbase/init.sql

@@ -0,0 +1,2 @@
+ALTER SYSTEM SET ob_vector_memory_limit_percentage = 30;
+

deploy/docker/docker-compose-pgvector.yml

@@ -69,18 +69,31 @@ services:
         # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程
         wait $$!
 
+  redis:
+    image: redis:7.2-alpine
+    container_name: redis
+    # ports:
+    # - 6379:6379
+    networks:
+      - fastgpt
+    restart: always
+    command: |
+      redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction
+    volumes:
+      - ./redis/data:/data
+
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.1-fix2 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.4 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.4 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.1-fix2 # git
+    image: ghcr.io/labring/fastgpt:v4.9.4 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.4 # 阿里云
     ports:
       - 3000:3000
     networks:
@@ -114,6 +127,8 @@ services:
       - MONGODB_URI=mongodb://myusername:mypassword@mongo:27017/fastgpt?authSource=admin
       # pg 连接参数
       - PG_URL=postgresql://username:password@pg:5432/postgres
+      # Redis 连接参数
+      - REDIS_URL=redis://default:mypassword@redis:6379
       # sandbox 地址
       - SANDBOX_URL=http://sandbox:3000
       # 日志等级: debug, info, warn, error
@@ -127,12 +142,14 @@ services:
       - ALLOWED_ORIGINS=
       # 是否开启IP限制，默认不开启
       - USE_IP_LIMIT=false
+      # 对话文件过期天数
+      - CHAT_FILE_EXPIRE_TIME=7
     volumes:
       - ./config.json:/app/data/config.json
 
   # AI Proxy
   aiproxy:
-    image: ghcr.io/labring/aiproxy:v0.1.3
+    image: ghcr.io/labring/aiproxy:v0.1.5
     # image: registry.cn-hangzhou.aliyuncs.com/labring/aiproxy:v0.1.3 # 阿里云
     container_name: aiproxy
     restart: unless-stopped

deploy/docker/docker-compose-zilliz.yml

@@ -51,17 +51,30 @@ services:
 
         # 等待docker-entrypoint.sh脚本执行的MongoDB服务进程
         wait $$!
+  redis:
+    image: redis:7.2-alpine
+    container_name: redis
+    # ports:
+    # - 6379:6379
+    networks:
+      - fastgpt
+    restart: always
+    command: |
+      redis-server --requirepass mypassword --loglevel warning --maxclients 10000 --appendonly yes --save 60 10 --maxmemory 4gb --maxmemory-policy noeviction
+    volumes:
+      - ./redis/data:/data
+
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.1-fix2 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.4 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.4 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.1-fix2 # git
+    image: ghcr.io/labring/fastgpt:v4.9.4 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1-fix2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.4 # 阿里云
     ports:
       - 3000:3000
     networks:
@@ -92,6 +105,8 @@ services:
       - FILE_TOKEN_KEY=filetoken
       # MongoDB 连接参数. 用户名myusername,密码mypassword。
       - MONGODB_URI=mongodb://myusername:mypassword@mongo:27017/fastgpt?authSource=admin
+      # Redis 连接参数
+      - REDIS_URI=redis://default:mypassword@redis:6379
       # zilliz 连接参数
       - MILVUS_ADDRESS=zilliz_cloud_address
       - MILVUS_TOKEN=zilliz_cloud_token
@@ -108,6 +123,8 @@ services:
       - ALLOWED_ORIGINS=
       # 是否开启IP限制，默认不开启
       - USE_IP_LIMIT=false
+      # 对话文件过期天数
+      - CHAT_FILE_EXPIRE_TIME=7
     volumes:
       - ./config.json:/app/data/config.json
 

docSite/content/zh-cn/docs/development/docker.md

@@ -135,6 +135,9 @@ curl -O https://raw.githubusercontent.com/labring/FastGPT/main/projects/app/data
 
 # pgvector 版本(测试推荐，简单快捷)
 curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-pgvector.yml
+# oceanbase 版本（需要将init.sql和docker-compose.yml放在同一个文件夹，方便挂载）
+# curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-oceanbase/docker-compose.yml
+# curl -o init.sql https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-oceanbase/init.sql
 # milvus 版本
 # curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-milvus.yml
 # zilliz 版本
@@ -151,6 +154,13 @@ curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/mai
 
 无需操作
 
+{{< /markdownify >}}
+{{< /tab >}}
+{{< tab tabName="Oceanbase版本" >}}
+{{< markdownify >}}
+
+无需操作
+
 {{< /markdownify >}}
 {{< /tab >}}
 {{< tab tabName="Milvus版本" >}}

docSite/content/zh-cn/docs/development/openapi/chat.md

@@ -18,12 +18,14 @@ weight: 852
 {{% alert icon="🤖 " context="success" %}}
 * 该接口的 API Key 需使用`应用特定的 key`，否则会报错。  
 
+<!-- * 对话现在有`v1`和`v2`两个接口，可以按需使用，v2 自 4.9.4 版本新增，v1 接口同时不再维护 -->
+
 * 有些包调用时，`BaseUrl`需要添加`v1`路径，有些不需要，如果出现404情况，可补充`v1`重试。
 {{% /alert %}}
 
 ## 请求简易应用和工作流
 
-对话接口兼容`GPT`的接口！如果你的项目使用的是标准的`GPT`官方接口，可以直接通过修改`BaseUrl`和 `Authorization`来访问 FastGpt 应用，不过需要注意下面几个规则：
+`v1`对话接口兼容`GPT`的接口！如果你的项目使用的是标准的`GPT`官方接口，可以直接通过修改`BaseUrl`和 `Authorization`来访问 FastGpt 应用，不过需要注意下面几个规则：
 
 {{% alert icon="🤖 " context="success" %}}
 * 传入的`model`，`temperature`等参数字段均无效，这些字段由编排决定，不会根据 API 参数改变。
@@ -65,7 +67,7 @@ curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
 {{< markdownify >}}
 
 * 仅`messages`有部分区别，其他参数一致。
-* 目前不支持上次文件，需上传到自己的对象存储中，获取对应的文件链接。
+* 目前不支持上传文件，需上传到自己的对象存储中，获取对应的文件链接。
 
 ```bash
 curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
@@ -116,14 +118,284 @@ curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
 - variables: 模块变量，一个对象，会替换模块中，输入框内容里的`{{key}}`
 {{% /alert %}}
 
+{{< /markdownify >}}
+{{< /tab >}}
+{{< /tabs >}}
 
+<!-- #### v2
+
+v1,v2 接口请求参数一致，仅请求地址不一样。
+
+{{< tabs tabTotal="3" >}}
+{{< tab tabName="基础请求示例" >}}
+{{< markdownify >}}
+
+```bash
+curl --location --request POST 'http://localhost:3000/api/v2/chat/completions' \
+--header 'Authorization: fastgpt-xxxxxx' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+    "chatId": "my_chatId",
+    "stream": false,
+    "detail": false,
+    "responseChatItemId": "my_responseChatItemId",
+    "variables": {
+        "uid": "asdfadsfasfd2323",
+        "name": "张三"
+    },
+    "messages": [
+        {
+            "role": "user",
+            "content": "你是谁"
+        }
+    ]
+}'
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="图片/文件请求示例" >}}
+{{< markdownify >}}
+
+* 仅`messages`有部分区别，其他参数一致。
+* 目前不支持上传文件，需上传到自己的对象存储中，获取对应的文件链接。
+
+```bash
+curl --location --request POST 'http://localhost:3000/api/v2/chat/completions' \
+--header 'Authorization: Bearer fastgpt-xxxxxx' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+    "chatId": "abcd",
+    "stream": false,
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "导演是谁"
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "图片链接"
+                    }
+                },
+                {
+                    "type": "file_url",
+                    "name": "文件名",
+                    "url": "文档链接，支持 txt md html word pdf ppt csv excel"
+                }
+            ]
+        }
+    ]
+}'
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="参数说明" >}}
+{{< markdownify >}}
+
+{{% alert context="info" %}}
+- headers.Authorization: Bearer {{apikey}}
+- chatId: string | undefined 。
+  - 为 `undefined` 时（不传入），不使用 FastGpt 提供的上下文功能，完全通过传入的 messages 构建上下文。
+  - 为`非空字符串`时，意味着使用 chatId 进行对话，自动从 FastGpt 数据库取历史记录，并使用 messages 数组最后一个内容作为用户问题，其余 message 会被忽略。请自行确保 chatId 唯一，长度小于250，通常可以是自己系统的对话框ID。
+- messages: 结构与 [GPT接口](https://platform.openai.com/docs/api-reference/chat/object) chat模式一致。
+- responseChatItemId: string | undefined 。如果传入，则会将该值作为本次对话的响应消息的 ID，FastGPT 会自动将该 ID 存入数据库。请确保，在当前`chatId`下，`responseChatItemId`是唯一的。
+- detail: 是否返回中间值（模块状态，响应的完整结果等），`stream模式`下会通过`event`进行区分，`非stream模式`结果保存在`responseData`中。
+- variables: 模块变量，一个对象，会替换模块中，输入框内容里的`{{key}}`
+{{% /alert %}}
 
 {{< /markdownify >}}
 {{< /tab >}}
 {{< /tabs >}}
 
+#### v1
+
+
+
 ### 响应
 
+#### v2
+
+v2 接口比起 v1，主要变变化在于：会在每个节点运行结束后及时返回 response，而不是等工作流结束后再统一返回。
+
+{{< tabs tabTotal="5" >}}
+{{< tab tabName="detail=false,stream=false 响应" >}}
+{{< markdownify >}}
+
+```json
+{
+    "id": "",
+    "model": "",
+    "usage": {
+        "prompt_tokens": 1,
+        "completion_tokens": 1,
+        "total_tokens": 1
+    },
+    "choices": [
+        {
+            "message": {
+                "role": "assistant",
+                "content": "我是一个人工智能助手，旨在回答问题和提供信息。如果你有任何问题或者需要帮助，随时问我！"
+            },
+            "finish_reason": "stop",
+            "index": 0
+        }
+    ]
+}
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="detail=false,stream=true 响应" >}}
+{{< markdownify >}}
+
+
+```bash
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":"你好"},"index":0,"finish_reason":null}]}
+
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":"！"},"index":0,"finish_reason":null}]}
+
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":"今天"},"index":0,"finish_reason":null}]}
+
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":"过得怎么样？"},"index":0,"finish_reason":null}]}
+
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":null},"index":0,"finish_reason":"stop"}]}
+
+data: [DONE]
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="detail=true,stream=false 响应" >}}
+{{< markdownify >}}
+
+```json
+{
+    "responseData": [
+        {
+            "id": "iSol79OFrBH1I9kC",
+            "nodeId": "448745",
+            "moduleName": "common:core.module.template.work_start",
+            "moduleType": "workflowStart",
+            "runningTime": 0
+        },
+        {
+            "id": "t1T94WCy6Su3BK4V",
+            "nodeId": "fjLpE3XPegmoGtbU",
+            "moduleName": "AI 对话",
+            "moduleType": "chatNode",
+            "runningTime": 1.46,
+            "totalPoints": 0,
+            "model": "GPT-4o-mini",
+            "tokens": 64,
+            "inputTokens": 10,
+            "outputTokens": 54,
+            "query": "你是谁",
+            "reasoningText": "",
+            "historyPreview": [
+                {
+                    "obj": "Human",
+                    "value": "你是谁"
+                },
+                {
+                    "obj": "AI",
+                    "value": "我是一个人工智能助手，旨在帮助回答问题和提供信息。如果你有任何问题或需要帮助，请告诉我！"
+                }
+            ],
+            "contextTotalLen": 2
+        }
+    ],
+    "newVariables": {
+
+    },
+    "id": "",
+    "model": "",
+    "usage": {
+        "prompt_tokens": 1,
+        "completion_tokens": 1,
+        "total_tokens": 1
+    },
+    "choices": [
+        {
+            "message": {
+                "role": "assistant",
+                "content": "我是一个人工智能助手，旨在帮助回答问题和提供信息。如果你有任何问题或需要帮助，请告诉我！"
+            },
+            "finish_reason": "stop",
+            "index": 0
+        }
+    ]
+}
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+
+{{< tab tabName="detail=true,stream=true 响应" >}}
+{{< markdownify >}}
+
+```bash
+event: flowNodeResponse
+data: {"id":"iYv2uA9rCWAtulWo","nodeId":"workflowStartNodeId","moduleName":"流程开始","moduleType":"workflowStart","runningTime":0}
+
+event: flowNodeStatus
+data: {"status":"running","name":"AI 对话"}
+
+event: answer
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":"你好"},"index":0,"finish_reason":null}]}
+
+event: answer
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":"！"},"index":0,"finish_reason":null}]}
+
+event: answer
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":"今天"},"index":0,"finish_reason":null}]}
+
+event: answer
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":"过得怎么样？"},"index":0,"finish_reason":null}]}
+
+event: flowNodeResponse
+data: {"id":"pVzLBF7M3Ol4n7s6","nodeId":"ixe20AHN3jy74pKf","moduleName":"AI 对话","moduleType":"chatNode","runningTime":1.48,"totalPoints":0.0042,"model":"Qwen-plus","tokens":28,"inputTokens":8,"outputTokens":20,"query":"你好","reasoningText":"","historyPreview":[{"obj":"Human","value":"你好"},{"obj":"AI","value":"你好！今天过得怎么样？"}],"contextTotalLen":2}
+
+event: answer
+data: {"id":"","object":"","created":0,"model":"","choices":[{"delta":{"role":"assistant","content":null},"index":0,"finish_reason":"stop"}]}
+
+event: answer
+data: [DONE]
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="event值" >}}
+{{< markdownify >}}
+
+event取值：
+
+- answer: 返回给客户端的文本（最终会算作回答）
+- fastAnswer: 指定回复返回给客户端的文本（最终会算作回答）
+- toolCall: 执行工具
+- toolParams: 工具参数
+- toolResponse: 工具返回
+- flowNodeStatus: 运行到的节点状态
+- flowNodeResponse: 单个节点详细响应
+- updateVariables: 更新变量
+- error: 报错
+
+{{< /markdownify >}}
+{{< /tab >}}
+{{< /tabs >}} 
+
+#### v1 -->
+
 {{< tabs tabTotal="5" >}}
 {{< tab tabName="detail=false,stream=false 响应" >}}
 {{< markdownify >}}
@@ -648,8 +920,6 @@ event取值：
 {{< /tab >}}
 {{< /tabs >}}
 
-
-
 # 对话 CRUD
 
 {{% alert icon="🤖 " context="success" %}}

docSite/content/zh-cn/docs/development/upgrading/491.md

@@ -39,7 +39,7 @@ curl --location --request POST 'https://{{host}}/api/admin/initv491' \
 3. API 知识库支持 PDF 增强解析。
 4. 邀请团队成员，改为邀请链接模式。
 5. 支持混合检索权重设置。
-6. 支持重排模型选择和权重设置，同时调整了知识库搜索权重计算方式，改成 搜索权重 + 重排权重，而不是向量检索权重+全文检索权重+重排权重。
+6. 支持重排模型选择和权重设置，同时调整了知识库搜索权重计算方式，改成 搜索权重 + 重排权重，而不是向量检索权重+全文检索权重+重排权重。会对检索结果有一定影响，可以通过调整相关权重来进行数据适配。
 
 ## ⚙️ 优化
 

docSite/content/zh-cn/docs/development/upgrading/492.md

@@ -1,5 +1,5 @@
 ---
-title: 'V4.9.2(进行中)'
+title: 'V4.9.2'
 description: 'FastGPT V4.9.2 更新说明'
 icon: 'upgrade'
 draft: false
@@ -8,6 +8,8 @@ weight: 798
 ---
 ## 更新指南
 
+可直接升级v4.9.3，v4.9.2存在一个工作流数据类型转化错误。
+
 ### 1. 做好数据库备份
 
 ### 2. SSO 迁移

docSite/content/zh-cn/docs/development/upgrading/493.md

@@ -0,0 +1,29 @@
+---
+title: 'V4.9.3'
+description: 'FastGPT V4.9.3 更新说明'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 797
+---
+
+## 更新指南
+
+### 1. 做好数据库备份
+
+### 2. 更新镜像
+
+- 更新 FastGPT 镜像 tag: v4.9.3
+- 更新 FastGPT 商业版镜像 tag: v4.9.3
+- Sandbox 镜像tag: v4.9.3
+- AIProxy 镜像tag: v0.1.5
+
+
+## 🚀 新增内容
+
+1. 工作流 debug 模式支持交互节点。
+2. 代码运行支持 Python3 代码。
+
+## 🐛 修复
+
+1. 工作流格式转化异常。

docSite/content/zh-cn/docs/development/upgrading/494.md

@@ -0,0 +1,66 @@
+---
+title: 'V4.9.4'
+description: 'FastGPT V4.9.4 更新说明'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 796
+---
+
+## 升级指南
+
+### 1. 做好数据备份
+
+### 2. 安装 Redis
+
+* docker 部署的用户，参考最新的 `docker-compose.yml` 文件增加 Redis 配置。增加一个 redis 容器，并配置`fastgpt`,`fastgpt-pro`的环境变量，增加 `REDIS_URL` 环境变量。
+* Sealos 部署的用户，在数据库里新建一个`redis`数据库，并复制`内网地址的 connection` 作为 `redis` 的链接串。然后配置`fastgpt`,`fastgpt-pro`的环境变量，增加 `REDIS_URL` 环境变量。
+
+| | | |
+| --- | --- |  --- |
+| ![](/imgs/sealos-redis1.png) | ![](/imgs/sealos-redis2.png) |  ![](/imgs/sealos-redis3.png) |
+
+### 3. 更新镜像 tag
+
+- 更新 FastGPT 镜像 tag: v4.9.4
+- 更新 FastGPT 商业版镜像 tag: v4.9.4
+- Sandbox 无需更新
+- AIProxy 无需更新
+
+### 4. 执行升级脚本
+
+该脚本仅需商业版用户执行。
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv494' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+**脚本功能**
+
+1. 更新站点同步定时器
+
+## 🚀 新增内容
+
+1. 集合数据训练状态展示
+2. SMTP 发送邮件插件
+3. BullMQ 消息队列。
+4. 利用 redis 进行部分数据缓存。
+5. 站点同步支持配置训练参数和增量同步。
+6. AI 对话/工具调用，增加返回模型 finish_reason 字段，便于追踪模型输出中断原因。
+7. 移动端语音输入交互调整
+
+## ⚙️ 优化
+
+1. Admin 模板渲染调整。
+2. 支持环境变量配置对话文件过期时间。
+3. MongoDB log 库可独立部署。
+
+## 🐛 修复
+
+1. 搜索应用/知识库时，无法点击目录进入下一层。
+2. 重新训练时，参数未成功初始化。
+3. package/service 部分请求在多 app 中不一致。

docSite/content/zh-cn/docs/development/upgrading/495.md

@@ -0,0 +1,28 @@
+---
+title: 'V4.9.5（进行中）'
+description: 'FastGPT V4.9.5 更新说明'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 795
+---
+
+
+## 🚀 新增内容
+
+1. 团队成员权限细分，可分别控制是否可创建在根目录应用/知识库以及 API Key
+2. 支持交互节点在嵌套工作流中使用。
+3. 团队成员操作日志。
+
+## ⚙️ 优化
+
+1. 繁体中文翻译。
+
+
+## 🐛 修复
+
+1. password 检测规则错误。
+2. 分享链接无法隐藏知识库检索结果。
+3. IOS 低版本正则兼容问题。
+4. 修复问答提取队列错误后，计数器未清零问题，导致问答提取队列失效。
+5. Debug 模式交互节点下一步可能造成死循环。

docSite/content/zh-cn/docs/guide/DialogBoxes/quoteList.md

@@ -0,0 +1,88 @@
+---
+title: '知识库引用分块阅读器'
+description: 'FastGPT 分块阅读器功能介绍'
+icon: 'description'
+draft: false
+toc: true
+weight: 480
+---
+在企业 AI 应用落地过程中，文档知识引用的精确性和透明度一直是用户关注的焦点。FastGPT 4.9.1 版本带来的知识库分块阅读器，巧妙解决了这一痛点，让 AI 引用不再是"黑盒"。
+
+# 为什么需要分块阅读器？
+
+传统的 AI 对话中，当模型引用企业知识库内容时，用户往往只能看到被引用的片段，无法获取完整语境，这给内容验证和深入理解带来了挑战。分块阅读器的出现，让用户可以在对话中直接查看引用内容的完整文档，并精确定位到引用位置，实现了引用的"可解释性"。
+
+## 传统引用体验的局限
+
+以往在知识库中上传文稿后，当我们在工作流中输入问题时，传统的引用方式只会展示引用到的分块，无法确认分块在文章中的上下文：
+
+| 问题 | 引用 |
+| --- | --- |
+| ![](/imgs/chunkReader1.png) | ![](/imgs/chunkReader2.jpg) |
+
+## FastGPT 分块阅读器：精准定位，无缝阅读
+
+而在 FastGPT 全新的分块式阅读器中，同样的知识库内容和问题，呈现方式发生了质的飞跃
+
+![](/imgs/chunkReader4.jpg) 
+
+当 AI 引用知识库内容时，用户只需点击引用链接，即可打开一个浮窗，呈现完整的原文内容，并通过醒目的高亮标记精确显示引用的文本片段。这既保证了回答的可溯源性，又提供了便捷的原文查阅体验。
+
+# 核心功能
+
+## 全文展示与定位
+
+"分块阅读器" 让用户能直观查看AI回答引用的知识来源。
+
+在对话界面中，当 AI 引用了知识库内容，系统会在回复下方展示出处信息。用户只需点击这些引用链接，即可打开一个优雅的浮窗，呈现完整的原文内容，并通过醒目的高亮标记精确显示 AI 引用的文本片段。
+
+这一设计既保证了回答的可溯源性，又提供了便捷的原文查阅体验，让用户能轻松验证AI回答的准确性和相关上下文。
+
+![](/imgs/chunkReader3.webp) 
+
+
+## 便捷引用导航
+
+分块阅读器右上角设计了简洁实用的导航控制，用户可以通过这对按钮轻松在多个引用间切换浏览。导航区还直观显示当前查看的引用序号及总引用数量（如 "7/10"），帮助用户随时了解浏览进度和引用内容的整体规模。
+
+![](imgs/chunkReader5.jpg)
+
+## 引用质量评分
+
+每条引用内容旁边都配有智能评分标签，直观展示该引用在所有知识片段中的相关性排名。用户只需将鼠标悬停在评分标签上，即可查看完整的评分详情，了解这段引用内容为何被AI选中以及其相关性的具体构成。
+
+![](imgs/chunkReader6.png)
+
+
+## 文档内容一键导出
+
+分块阅读器贴心配备了内容导出功能，让有效信息不再流失。只要用户拥有相应知识库的阅读权限，便可通过简单点击将引用涉及的全文直接保存到本地设备。
+
+![](imgs/chunkReader7.jpg)
+
+# 进阶特性
+
+## 灵活的可见度控制
+
+FastGPT提供灵活的引用可见度设置，让知识共享既开放又安全。以免登录链接为例，管理员可精确控制外部访问者能看到的信息范围。
+
+当设置为"仅引用内容可见"时，外部用户点击引用链接将只能查看 AI 引用的特定文本片段，而非完整原文档。如图所示，分块阅读器此时智能调整显示模式，仅呈现相关引用内容。
+
+|  |  |
+| --- | --- |
+| ![](/imgs/chunkReader8.png) | ![](/imgs/chunkReader9.jpg) |
+
+## 即时标注优化
+
+在浏览过程中，授权用户可以直接对引用内容进行即时标注和修正，系统会智能处理这些更新而不打断当前的对话体验。所有修改过的内容会通过醒目的"已更新"标签清晰标识，既保证了引用的准确性，又维持了对话历史的完整性。
+
+这一无缝的知识优化流程特别适合团队协作场景，让知识库能在实际使用过程中持续进化，确保AI回答始终基于最新、最准确的信息源。
+
+## 智能文档性能优化
+
+面对现实业务中可能包含成千上万分块的超长文档，FastGPT采用了先进的性能优化策略，确保分块阅读器始终保持流畅响应。
+
+系统根据引用相关性排序和数据库索引进行智能加载管理，实现了"按需渲染"机制——根据索引排序和数据库 id，只有当用户实际需要查看的内容才会被加载到内存中。这意味着无论是快速跳转到特定引用，还是自然滚动浏览文档，都能获得丝滑的用户体验，不会因为文档体积庞大而出现卡顿或延迟。
+
+这一技术优化使FastGPT能够轻松应对企业级的大规模知识库场景，让即使是包含海量信息的专业文档也能高效展示和查阅。
+

docSite/content/zh-cn/docs/guide/admin/sso.md

@@ -61,11 +61,12 @@ FastGPT-SSO-Service 是为了聚合不同来源的 SSO 和成员同步接口，
 
 #### 1. 配置环境变量
 
-环境变量中的 `EXTERNAL_USER_SERVICE_BASE_URL` 为内网地址，例如上述例子中的配置，环境变量应该设置为
+环境变量中的 `EXTERNAL_USER_SYSTEM_BASE_URL` 为内网地址，例如上述例子中的配置，环境变量应该设置为
 
 ```yaml
-EXTERNAL_USER_SERVICE_BASE_URL=http://fastgpt-sso:3000
+env:
-EXTERNAL_USER_SERVICE_AUTH_TOKEN=xxxxx
+   - EXTERNAL_USER_SYSTEM_BASE_URL=http://fastgpt-sso:3000
+   - EXTERNAL_USER_SYSTEM_AUTH_TOKEN=xxxxx
 ```
 
 #### 2. 在商业版后台配置按钮文字，图标等。
@@ -89,7 +90,8 @@ EXTERNAL_USER_SERVICE_AUTH_TOKEN=xxxxx
 设置 fastgpt-pro 环境变量则可开启自动成员同步
 
 ```bash
-SYNC_MEMBER_CRON="0 0 * * *" # Cron 表达式，每天 0 点执行
+env:
+   - "SYNC_MEMBER_CRON=0 0 * * *" # Cron 表达式，每天 0 点执行
 ```
 
 ## 内置的通用协议/IM 配置示例

docSite/content/zh-cn/docs/intro.md

@@ -7,7 +7,7 @@ toc: true
 weight: -10
 ---
 
-FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开箱即用的数据处理、模型调用等能力。同时可以通过 Flow 可视化进行工作流编排，从而实现复杂的问答场景！
+FastGPT 是一个AI Agent 构建平台，提供开箱即用的数据处理、模型调用等能力，同时可以通过 Flow 可视化进行工作流编排，从而实现复杂的应用场景！
 
 {{% alert icon="🤖 " context="success" %}}
 FastGPT 在线使用：[https://tryfastgpt.ai](https://tryfastgpt.ai)

env.d.ts

@@ -0,0 +1,40 @@
+declare global {
+  namespace NodeJS {
+    interface ProcessEnv {
+      LOG_DEPTH: string;
+      DEFAULT_ROOT_PSW: string;
+      DB_MAX_LINK: string;
+      TOKEN_KEY: string;
+      FILE_TOKEN_KEY: string;
+      ROOT_KEY: string;
+      OPENAI_BASE_URL: string;
+      CHAT_API_KEY: string;
+      AIPROXY_API_ENDPOINT: string;
+      AIPROXY_API_TOKEN: string;
+      MULTIPLE_DATA_TO_BASE64: string;
+      MONGODB_URI: string;
+      MONGODB_LOG_URI?: string;
+      PG_URL: string;
+      OCEANBASE_URL: string;
+      MILVUS_ADDRESS: string;
+      MILVUS_TOKEN: string;
+      SANDBOX_URL: string;
+      PRO_URL: string;
+      FE_DOMAIN: string;
+      FILE_DOMAIN: string;
+      NEXT_PUBLIC_BASE_URL: string;
+      LOG_LEVEL: string;
+      STORE_LOG_LEVEL: string;
+      USE_IP_LIMIT: string;
+      WORKFLOW_MAX_RUN_TIMES: string;
+      WORKFLOW_MAX_LOOP_TIMES: string;
+      CHECK_INTERNAL_IP: string;
+      CHAT_LOG_URL: string;
+      CHAT_LOG_INTERVAL: string;
+      CHAT_LOG_SOURCE_ID_PREFIX: string;
+      ALLOWED_ORIGINS: string;
+    }
+  }
+}
+
+export {};

package.json

@@ -12,27 +12,29 @@
     "previewIcon": "node ./scripts/icon/index.js",
     "api:gen": "tsc ./scripts/openapi/index.ts && node ./scripts/openapi/index.js && npx @redocly/cli build-docs ./scripts/openapi/openapi.json -o ./projects/app/public/openapi/index.html",
     "create:i18n": "node ./scripts/i18n/index.js",
-    "test": "vitest run --exclude 'test/cases/spec'",
+    "test": "vitest run",
-    "test:all": "vitest run",
     "test:workflow": "vitest run workflow"
   },
   "devDependencies": {
     "@chakra-ui/cli": "^2.4.1",
-    "@vitest/coverage-v8": "^3.0.2",
+    "@vitest/coverage-v8": "^3.0.9",
     "husky": "^8.0.3",
     "i18next": "23.16.8",
     "lint-staged": "^13.3.0",
     "next-i18next": "15.4.2",
     "prettier": "3.2.4",
     "react-i18next": "14.1.2",
-    "vitest": "^3.0.2",
+    "vitest": "^3.0.9",
-    "vitest-mongodb": "^1.0.1",
+    "mongodb-memory-server": "^10.1.4",
     "zhlint": "^0.7.4"
   },
   "lint-staged": {
     "./**/**/*.{ts,tsx,scss}": "npm run format-code",
     "./docSite/**/**/*.md": "npm run format-doc"
   },
+  "resolutions": {
+    "mdast-util-gfm-autolink-literal": "2.0.0"
+  },
   "engines": {
     "node": ">=18.16.0",
     "pnpm": ">=9.0.0"

packages/global/common/system/types/index.d.ts

@@ -118,11 +118,12 @@ export type SystemEnvType = {
   oneapiUrl?: string;
   chatApiKey?: string;
 
-  customPdfParse?: {
+  customPdfParse?: customPdfParseType;
-    url?: string;
+};
-    key?: string;
+
-
+export type customPdfParseType = {
-    doc2xKey?: string;
+  url?: string;
-    price?: number; // n points/1 page
+  key?: string;
-  };
+  doc2xKey?: string;
+  price?: number;
 };

packages/global/core/ai/constants.ts

@@ -1,3 +1,5 @@
+import { i18nT } from '../../../web/i18n/utils';
+
 export enum ChatCompletionRequestMessageRoleEnum {
   'System' = 'system',
   'User' = 'user',
@@ -28,3 +30,13 @@ export enum EmbeddingTypeEnm {
   query = 'query',
   db = 'db'
 }
+
+export const completionFinishReasonMap = {
+  close: i18nT('chat:completion_finish_close'),
+  stop: i18nT('chat:completion_finish_stop'),
+  length: i18nT('chat:completion_finish_length'),
+  tool_calls: i18nT('chat:completion_finish_tool_calls'),
+  content_filter: i18nT('chat:completion_finish_content_filter'),
+  function_call: i18nT('chat:completion_finish_function_call'),
+  null: i18nT('chat:completion_finish_null')
+};

packages/global/core/ai/type.d.ts

@@ -73,6 +73,15 @@ export type ChatCompletionMessageFunctionCall =
 export type StreamChatType = Stream<openai.Chat.Completions.ChatCompletionChunk>;
 export type UnStreamChatType = openai.Chat.Completions.ChatCompletion;
 
+export type CompletionFinishReason =
+  | 'close'
+  | 'stop'
+  | 'length'
+  | 'tool_calls'
+  | 'content_filter'
+  | 'function_call'
+  | null;
+
 export default openai;
 export * from 'openai';
 

packages/global/core/chat/utils.ts

@@ -77,6 +77,13 @@ export const getHistoryPreview = (
   });
 };
 
+export const filterModuleTypeList: any[] = [
+  FlowNodeTypeEnum.pluginModule,
+  FlowNodeTypeEnum.datasetSearchNode,
+  FlowNodeTypeEnum.tools,
+  FlowNodeTypeEnum.pluginOutput
+];
+
 export const filterPublicNodeResponseData = ({
   flowResponses = [],
   responseDetail = false
@@ -87,12 +94,6 @@ export const filterPublicNodeResponseData = ({
   const filedList = responseDetail
     ? ['quoteList', 'moduleType', 'pluginOutput', 'runningTime']
     : ['moduleType', 'pluginOutput', 'runningTime'];
-  const filterModuleTypeList: any[] = [
-    FlowNodeTypeEnum.pluginModule,
-    FlowNodeTypeEnum.datasetSearchNode,
-    FlowNodeTypeEnum.tools,
-    FlowNodeTypeEnum.pluginOutput
-  ];
 
   return flowResponses
     .filter((item) => filterModuleTypeList.includes(item.moduleType))

packages/global/core/dataset/api.d.ts

@@ -15,7 +15,6 @@ export type DatasetUpdateBody = {
   name?: string;
   avatar?: string;
   intro?: string;
-  status?: DatasetSchemaType['status'];
 
   agentModel?: string;
   vlmModel?: string;
@@ -26,6 +25,7 @@ export type DatasetUpdateBody = {
   apiServer?: DatasetSchemaType['apiServer'];
   yuqueServer?: DatasetSchemaType['yuqueServer'];
   feishuServer?: DatasetSchemaType['feishuServer'];
+  chunkSettings?: DatasetSchemaType['chunkSettings'];
 
   // sync schedule
   autoSync?: boolean;
@@ -141,7 +141,6 @@ export type PushDatasetDataChunkProps = {
 
 export type PostWebsiteSyncParams = {
   datasetId: string;
-  billId: string;
 };
 
 export type PushDatasetDataProps = {

packages/global/core/dataset/constants.ts

@@ -50,7 +50,9 @@ export const DatasetTypeMap = {
 
 export enum DatasetStatusEnum {
   active = 'active',
-  syncing = 'syncing'
+  syncing = 'syncing',
+  waiting = 'waiting',
+  error = 'error'
 }
 export const DatasetStatusMap = {
   [DatasetStatusEnum.active]: {
@@ -58,6 +60,12 @@ export const DatasetStatusMap = {
   },
   [DatasetStatusEnum.syncing]: {
     label: i18nT('common:core.dataset.status.syncing')
+  },
+  [DatasetStatusEnum.waiting]: {
+    label: i18nT('common:core.dataset.status.waiting')
+  },
+  [DatasetStatusEnum.error]: {
+    label: i18nT('dataset:status_error')
   }
 };
 

packages/global/core/dataset/type.d.ts

@@ -17,6 +17,20 @@ import { SourceMemberType } from 'support/user/type';
 import { DatasetDataIndexTypeEnum } from './data/constants';
 import { ChunkSettingModeEnum } from './constants';
 
+export type ChunkSettingsType = {
+  trainingType: DatasetCollectionDataProcessModeEnum;
+  autoIndexes?: boolean;
+  imageIndex?: boolean;
+
+  chunkSettingMode?: ChunkSettingModeEnum;
+  chunkSplitMode?: DataChunkSplitModeEnum;
+
+  chunkSize?: number;
+  indexSize?: number;
+  chunkSplitter?: string;
+  qaPrompt?: string;
+};
+
 export type DatasetSchemaType = {
   _id: string;
   parentId?: string;
@@ -29,7 +43,6 @@ export type DatasetSchemaType = {
   name: string;
   intro: string;
   type: `${DatasetTypeEnum}`;
-  status: `${DatasetStatusEnum}`;
 
   vectorModel: string;
   agentModel: string;
@@ -39,14 +52,16 @@ export type DatasetSchemaType = {
     url: string;
     selector: string;
   };
+
+  chunkSettings?: ChunkSettingsType;
+
   inheritPermission: boolean;
   apiServer?: APIFileServer;
   feishuServer?: FeishuServer;
   yuqueServer?: YuqueServer;
 
-  autoSync?: boolean;
-
   // abandon
+  autoSync?: boolean;
   externalReadUrl?: string;
   defaultPermission?: number;
 };
@@ -163,6 +178,7 @@ export type DatasetTrainingSchemaType = {
   weight: number;
   indexes: Omit<DatasetDataIndexItemType, 'dataId'>[];
   retryCount: number;
+  errorMsg?: string;
 };
 
 export type CollectionWithDatasetType = DatasetCollectionSchemaType & {
@@ -192,6 +208,8 @@ export type DatasetListItemType = {
 };
 
 export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel' | 'vlmModel'> & {
+  status: `${DatasetStatusEnum}`;
+  errorMsg?: string;
   vectorModel: EmbeddingModelItemType;
   agentModel: LLMModelItemType;
   vlmModel?: LLMModelItemType;
@@ -216,6 +234,7 @@ export type DatasetCollectionItemType = CollectionWithDatasetType & {
   file?: DatasetFileSchema;
   permission: DatasetPermission;
   indexAmount: number;
+  errorCount?: number;
 };
 
 /* ================= data ===================== */

packages/global/core/workflow/runtime/constants.ts

@@ -5,6 +5,7 @@ export enum SseResponseEventEnum {
   answer = 'answer', // animation stream
   fastAnswer = 'fastAnswer', // direct answer text, not animation
   flowNodeStatus = 'flowNodeStatus', // update node status
+  flowNodeResponse = 'flowNodeResponse', // node response
 
   toolCall = 'toolCall', // tool start
   toolParams = 'toolParams', // tool params return

packages/global/core/workflow/runtime/type.d.ts

@@ -22,7 +22,8 @@ import { UserSelectOptionType } from '../template/system/userSelect/type';
 import { WorkflowResponseType } from '../../../../service/core/workflow/dispatch/type';
 import { AiChatQuoteRoleType } from '../template/system/aiChat/type';
 import { LafAccountType, OpenaiAccountType } from '../../../support/user/team/type';
-
+import { CompletionFinishReason } from '../../ai/type';
+import { WorkflowInteractiveResponseType } from '../template/system/interactive/type';
 export type ExternalProviderType = {
   openaiAccount?: OpenaiAccountType;
   externalWorkflowVariables?: Record<string, string>;
@@ -40,6 +41,7 @@ export type ChatDispatchProps = {
     id: string; // May be the id of the system plug-in (cannot be used directly to look up the table)
     teamId: string;
     tmbId: string; // App tmbId
+    isChildApp?: boolean;
   };
   runningUserInfo: {
     teamId: string;
@@ -53,11 +55,14 @@ export type ChatDispatchProps = {
   variables: Record<string, any>; // global variable
   query: UserChatItemValueItemType[]; // trigger query
   chatConfig: AppSchema['chatConfig'];
+  lastInteractive?: WorkflowInteractiveResponseType; // last interactive response
   stream: boolean;
   maxRunTimes: number;
   isToolCall?: boolean;
   workflowStreamResponse?: WorkflowResponseType;
   workflowDispatchDeep?: number;
+  version?: 'v1' | 'v2';
+  responseDetail?: boolean;
 };
 
 export type ModuleDispatchProps<T> = ChatDispatchProps & {
@@ -128,6 +133,7 @@ export type DispatchNodeResponseType = {
     obj: `${ChatRoleEnum}`;
     value: string;
   }[]; // completion context array. history will slice
+  finishReason?: CompletionFinishReason;
 
   // dataset search
   similarity?: number;

packages/global/core/workflow/runtime/utils.ts

@@ -10,7 +10,19 @@ import { FlowNodeOutputItemType, ReferenceValueType } from '../type/io';
 import { ChatItemType, NodeOutputItemType } from '../../../core/chat/type';
 import { ChatItemValueTypeEnum, ChatRoleEnum } from '../../../core/chat/constants';
 import { replaceVariable, valToStr } from '../../../common/string/tools';
+import {
+  InteractiveNodeResponseType,
+  WorkflowInteractiveResponseType
+} from '../template/system/interactive/type';
 
+export const extractDeepestInteractive = (
+  interactive: WorkflowInteractiveResponseType
+): WorkflowInteractiveResponseType => {
+  if (interactive?.type === 'childrenInteractive' && interactive.params?.childrenResponse) {
+    return extractDeepestInteractive(interactive.params.childrenResponse);
+  }
+  return interactive;
+};
 export const getMaxHistoryLimitFromNodes = (nodes: StoreNodeItemType[]): number => {
   let limit = 10;
   nodes.forEach((node) => {
@@ -34,7 +46,9 @@ export const getMaxHistoryLimitFromNodes = (nodes: StoreNodeItemType[]): number
   1. Get the interactive data
   2. Check that the workflow starts at the interaction node
 */
-export const getLastInteractiveValue = (histories: ChatItemType[]) => {
+export const getLastInteractiveValue = (
+  histories: ChatItemType[]
+): WorkflowInteractiveResponseType | undefined => {
   const lastAIMessage = [...histories].reverse().find((item) => item.obj === ChatRoleEnum.AI);
 
   if (lastAIMessage) {
@@ -45,7 +59,11 @@ export const getLastInteractiveValue = (histories: ChatItemType[]) => {
       lastValue.type !== ChatItemValueTypeEnum.interactive ||
       !lastValue.interactive
     ) {
-      return null;
+      return;
+    }
+
+    if (lastValue.interactive.type === 'childrenInteractive') {
+      return lastValue.interactive;
     }
 
     // Check is user select
@@ -62,38 +80,29 @@ export const getLastInteractiveValue = (histories: ChatItemType[]) => {
     }
   }
 
-  return null;
+  return;
 };
 
 export const initWorkflowEdgeStatus = (
   edges: StoreEdgeItemType[],
-  histories?: ChatItemType[]
+  lastInteractive?: WorkflowInteractiveResponseType
 ): RuntimeEdgeItemType[] => {
-  // If there is a history, use the last interactive value
+  if (lastInteractive) {
-  if (histories && histories.length > 0) {
+    const memoryEdges = lastInteractive.memoryEdges || [];
-    const memoryEdges = getLastInteractiveValue(histories)?.memoryEdges;
-
     if (memoryEdges && memoryEdges.length > 0) {
       return memoryEdges;
     }
   }
 
-  return (
+  return edges?.map((edge) => ({ ...edge, status: 'waiting' })) || [];
-    edges?.map((edge) => ({
-      ...edge,
-      status: 'waiting'
-    })) || []
-  );
 };
 
 export const getWorkflowEntryNodeIds = (
   nodes: (StoreNodeItemType | RuntimeNodeItemType)[],
-  histories?: ChatItemType[]
+  lastInteractive?: WorkflowInteractiveResponseType
 ) => {
-  // If there is a history, use the last interactive entry node
+  if (lastInteractive) {
-  if (histories && histories.length > 0) {
+    const entryNodeIds = lastInteractive.entryNodeIds || [];
-    const entryNodeIds = getLastInteractiveValue(histories)?.entryNodeIds;
-
     if (Array.isArray(entryNodeIds) && entryNodeIds.length > 0) {
       return entryNodeIds;
     }
@@ -396,10 +405,10 @@ export const textAdaptGptResponse = ({
 
 /* Update runtimeNode's outputs with interactive data from history */
 export function rewriteNodeOutputByHistories(
-  histories: ChatItemType[],
+  runtimeNodes: RuntimeNodeItemType[],
-  runtimeNodes: RuntimeNodeItemType[]
+  lastInteractive?: InteractiveNodeResponseType
 ) {
-  const interactive = getLastInteractiveValue(histories);
+  const interactive = lastInteractive;
   if (!interactive?.nodeOutputs) {
     return runtimeNodes;
   }

packages/global/core/workflow/template/system/interactive/type.d.ts

@@ -1,6 +1,5 @@
 import type { NodeOutputItemType } from '../../../../chat/type';
 import type { FlowNodeOutputItemType } from '../../../type/io';
-import type { RuntimeEdgeItemType } from '../../../runtime/type';
 import { FlowNodeInputTypeEnum } from 'core/workflow/node/constant';
 import { WorkflowIOValueTypeEnum } from 'core/workflow/constants';
 import type { ChatCompletionMessageParam } from '../../../../ai/type';
@@ -9,7 +8,6 @@ type InteractiveBasicType = {
   entryNodeIds: string[];
   memoryEdges: RuntimeEdgeItemType[];
   nodeOutputs: NodeOutputItemType[];
-
   toolParams?: {
     entryNodeIds: string[]; // 记录工具中，交互节点的 Id，而不是起始工作流的入口
     memoryMessages: ChatCompletionMessageParam[]; // 这轮工具中，产生的新的 messages
@@ -23,6 +21,13 @@ type InteractiveNodeType = {
   nodeOutputs?: NodeOutputItemType[];
 };
 
+type ChildrenInteractive = InteractiveNodeType & {
+  type: 'childrenInteractive';
+  params: {
+    childrenResponse?: WorkflowInteractiveResponseType;
+  };
+};
+
 export type UserSelectOptionItemType = {
   key: string;
   value: string;
@@ -62,5 +67,9 @@ type UserInputInteractive = InteractiveNodeType & {
     submitted?: boolean;
   };
 };
-export type InteractiveNodeResponseType = UserSelectInteractive | UserInputInteractive;
+
+export type InteractiveNodeResponseType =
+  | UserSelectInteractive
+  | UserInputInteractive
+  | ChildrenInteractive;
 export type WorkflowInteractiveResponseType = InteractiveBasicType & InteractiveNodeResponseType;

packages/global/package.json

@@ -10,7 +10,7 @@
     "js-yaml": "^4.1.0",
     "jschardet": "3.1.1",
     "nanoid": "^5.1.3",
-    "next": "14.2.25",
+    "next": "14.2.26",
     "openai": "4.61.0",
     "openapi-types": "^12.1.3",
     "json5": "^2.2.3",

packages/global/support/operationLog/constants.ts

@@ -0,0 +1,14 @@
+export enum OperationLogEventEnum {
+  LOGIN = 'LOGIN',
+  CREATE_INVITATION_LINK = 'CREATE_INVITATION_LINK',
+  JOIN_TEAM = 'JOIN_TEAM',
+  CHANGE_MEMBER_NAME = 'CHANGE_MEMBER_NAME',
+  KICK_OUT_TEAM = 'KICK_OUT_TEAM',
+  CREATE_DEPARTMENT = 'CREATE_DEPARTMENT',
+  CHANGE_DEPARTMENT = 'CHANGE_DEPARTMENT',
+  DELETE_DEPARTMENT = 'DELETE_DEPARTMENT',
+  RELOCATE_DEPARTMENT = 'RELOCATE_DEPARTMENT',
+  CREATE_GROUP = 'CREATE_GROUP',
+  DELETE_GROUP = 'DELETE_GROUP',
+  ASSIGN_PERMISSION = 'ASSIGN_PERMISSION'
+}

packages/global/support/operationLog/type.d.ts

@@ -0,0 +1,19 @@
+import { SourceMemberType } from '../user/type';
+import { OperationLogEventEnum } from './constants';
+
+export type OperationLogSchema = {
+  _id: string;
+  tmbId: string;
+  teamId: string;
+  timestamp: Date;
+  event: `${OperationLogEventEnum}`;
+  metadata?: Record<string, string>;
+};
+
+export type OperationListItemType = {
+  _id: string;
+  sourceMember: SourceMemberType;
+  event: `${OperationLogEventEnum}`;
+  timestamp: Date;
+  metadata: Record<string, string>;
+};

packages/global/support/permission/collaborator.d.ts

@@ -13,12 +13,15 @@ export type CollaboratorItemType = {
   orgId: string;
 }>;
 
-export type UpdateClbPermissionProps = {
+export type UpdateClbPermissionProps<addOnly = false> = {
   members?: string[];
   groups?: string[];
   orgs?: string[];
-  permission: PermissionValueType;
+} & (addOnly extends true
-};
+  ? {}
+  : {
+      permission: PermissionValueType;
+    });
 
 export type DeletePermissionQuery = RequireOnlyOne<{
   tmbId?: string;

packages/global/support/permission/controller.ts

@@ -5,15 +5,16 @@ export type PerConstructPros = {
   per?: PermissionValueType;
   isOwner?: boolean;
   permissionList?: PermissionListType;
+  childUpdatePermissionCallback?: () => void;
 };
 
 // the Permission helper class
 export class Permission {
   value: PermissionValueType;
-  isOwner: boolean;
+  isOwner: boolean = false;
-  hasManagePer: boolean;
+  hasManagePer: boolean = false;
-  hasWritePer: boolean;
+  hasWritePer: boolean = false;
-  hasReadPer: boolean;
+  hasReadPer: boolean = false;
   _permissionList: PermissionListType;
 
   constructor(props?: PerConstructPros) {
@@ -24,11 +25,8 @@ export class Permission {
       this.value = per;
     }
 
-    this.isOwner = isOwner;
     this._permissionList = permissionList;
-    this.hasManagePer = this.checkPer(this._permissionList['manage'].value);
+    this.updatePermissions();
-    this.hasWritePer = this.checkPer(this._permissionList['write'].value);
-    this.hasReadPer = this.checkPer(this._permissionList['read'].value);
   }
 
   // add permission(s)
@@ -68,10 +66,21 @@ export class Permission {
     return (this.value & perm) === perm;
   }
 
+  private updatePermissionCallback?: () => void;
+  setUpdatePermissionCallback(callback: () => void) {
+    callback();
+    this.updatePermissionCallback = callback;
+  }
+
   private updatePermissions() {
     this.isOwner = this.value === OwnerPermissionVal;
     this.hasManagePer = this.checkPer(this._permissionList['manage'].value);
     this.hasWritePer = this.checkPer(this._permissionList['write'].value);
     this.hasReadPer = this.checkPer(this._permissionList['read'].value);
+    this.updatePermissionCallback?.();
+  }
+
+  toBinary() {
+    return this.value.toString(2);
   }
 }

packages/global/support/permission/memberGroup/type.d.ts

@@ -17,23 +17,23 @@ type GroupMemberSchemaType = {
   role: `${GroupMemberRole}`;
 };
 
-type MemberGroupListItemType<T extends boolean | undefined> = MemberGroupSchemaType & {
+type MemberGroupListItemType<WithMembers extends boolean | undefined> = MemberGroupSchemaType & {
-  members: T extends true
+  members: WithMembers extends true
     ? {
         tmbId: string;
         name: string;
         avatar: string;
       }[]
     : undefined;
-  count: T extends true ? number : undefined;
+  count: WithMembers extends true ? number : undefined;
-  owner?: T extends true
+  owner?: WithMembers extends true
     ? {
         tmbId: string;
         name: string;
         avatar: string;
       }
     : undefined;
-  permission: T extends true ? Permission : undefined;
+  permission: WithMembers extends true ? Permission : undefined;
 };
 
 type GroupMemberItemType = {

packages/global/support/permission/user/constant.ts

@@ -1,22 +1,50 @@
 import { PermissionKeyEnum } from '../constant';
 import { PermissionListType } from '../type';
 import { PermissionList } from '../constant';
-export const TeamPermissionList: PermissionListType = {
+import { i18nT } from '../../../../web/i18n/utils';
+export enum TeamPermissionKeyEnum {
+  appCreate = 'appCreate',
+  datasetCreate = 'datasetCreate',
+  apikeyCreate = 'apikeyCreate'
+}
+
+export const TeamPermissionList: PermissionListType<TeamPermissionKeyEnum> = {
   [PermissionKeyEnum.read]: {
     ...PermissionList[PermissionKeyEnum.read],
-    value: 0b100
+    value: 0b000100
   },
   [PermissionKeyEnum.write]: {
     ...PermissionList[PermissionKeyEnum.write],
-    value: 0b010
+    value: 0b000010
   },
   [PermissionKeyEnum.manage]: {
     ...PermissionList[PermissionKeyEnum.manage],
-    value: 0b001
+    value: 0b000001
+  },
+  [TeamPermissionKeyEnum.appCreate]: {
+    checkBoxType: 'multiple',
+    description: '',
+    name: i18nT('account_team:permission_appCreate'),
+    value: 0b001000
+  },
+  [TeamPermissionKeyEnum.datasetCreate]: {
+    checkBoxType: 'multiple',
+    description: '',
+    name: i18nT('account_team:permission_datasetCreate'),
+    value: 0b010000
+  },
+  [TeamPermissionKeyEnum.apikeyCreate]: {
+    checkBoxType: 'multiple',
+    description: '',
+    name: i18nT('account_team:permission_apikeyCreate'),
+    value: 0b100000
   }
 };
 
 export const TeamReadPermissionVal = TeamPermissionList['read'].value;
 export const TeamWritePermissionVal = TeamPermissionList['write'].value;
 export const TeamManagePermissionVal = TeamPermissionList['manage'].value;
+export const TeamAppCreatePermissionVal = TeamPermissionList['appCreate'].value;
+export const TeamDatasetCreatePermissionVal = TeamPermissionList['datasetCreate'].value;
+export const TeamApikeyCreatePermissionVal = TeamPermissionList['apikeyCreate'].value;
 export const TeamDefaultPermissionVal = TeamReadPermissionVal;

packages/global/support/permission/user/controller.ts

@@ -1,7 +1,17 @@
 import { PerConstructPros, Permission } from '../controller';
-import { TeamDefaultPermissionVal, TeamPermissionList } from './constant';
+import {
+  TeamApikeyCreatePermissionVal,
+  TeamAppCreatePermissionVal,
+  TeamDatasetCreatePermissionVal,
+  TeamDefaultPermissionVal,
+  TeamPermissionList
+} from './constant';
 
 export class TeamPermission extends Permission {
+  hasAppCreatePer: boolean = false;
+  hasDatasetCreatePer: boolean = false;
+  hasApikeyCreatePer: boolean = false;
+
   constructor(props?: PerConstructPros) {
     if (!props) {
       props = {
@@ -12,5 +22,11 @@ export class TeamPermission extends Permission {
     }
     props.permissionList = TeamPermissionList;
     super(props);
+
+    this.setUpdatePermissionCallback(() => {
+      this.hasAppCreatePer = this.checkPer(TeamAppCreatePermissionVal);
+      this.hasDatasetCreatePer = this.checkPer(TeamDatasetCreatePermissionVal);
+      this.hasApikeyCreatePer = this.checkPer(TeamApikeyCreatePermissionVal);
+    });
   }
 }

packages/plugins/package.json

@@ -5,6 +5,7 @@
   "dependencies": {
     "cheerio": "1.0.0-rc.12",
     "@types/pg": "^8.6.6",
+    "@types/nodemailer": "^6.4.17",
     "axios": "^1.8.2",
     "duck-duck-scrape": "^2.2.5",
     "echarts": "5.4.1",
@@ -13,6 +14,7 @@
     "mssql": "^11.0.1",
     "mysql2": "^3.11.3",
     "json5": "^2.2.3",
+    "nodemailer": "^6.10.0",
     "pg": "^8.10.0",
     "wikijs": "^6.4.1"
   },

packages/plugins/register.ts

@@ -29,7 +29,8 @@ const packagePluginList = [
   'databaseConnection',
   'Doc2X',
   'Doc2X/PDF2text',
-  'searchXNG'
+  'searchXNG',
+  'smtpEmail'
 ];
 
 export const list = [...staticPluginList, ...packagePluginList];

packages/plugins/src/databaseConnection/index.ts

@@ -34,7 +34,8 @@ const main = async ({
         port: parseInt(port, 10),
         database: databaseName,
         user,
-        password
+        password,
+        connectionTimeoutMillis: 30000
       });
 
       await client.connect();
@@ -47,7 +48,8 @@ const main = async ({
         port: parseInt(port, 10),
         database: databaseName,
         user,
-        password
+        password,
+        connectTimeout: 30000
       });
 
       const [rows] = await connection.execute(sql);

packages/plugins/src/smtpEmail/index.ts

@@ -0,0 +1,122 @@
+import { getErrText } from '@fastgpt/global/common/error/utils';
+import nodemailer from 'nodemailer';
+
+interface Props {
+  // SMTP配置
+  smtpHost: string;
+  smtpPort: string;
+  SSL: boolean;
+  smtpUser: string;
+  smtpPass: string;
+  fromName?: string;
+  // 邮件参数
+  to: string;
+  subject: string;
+  content: string;
+  cc?: string;
+  bcc?: string;
+  attachments?: string;
+}
+
+interface Response {
+  success: boolean;
+  messageId?: string;
+  error?: string;
+}
+
+const validateEmail = (email: string) => {
+  const regex = /^[^\s@]+@[^\s@]+\.[^\s@]+$/;
+  return regex.test(email);
+};
+
+const validateEmails = (emails: string) => {
+  return emails.split(',').every((email) => validateEmail(email.trim()));
+};
+
+const main = async ({
+  smtpHost,
+  smtpPort,
+  SSL,
+  smtpUser,
+  smtpPass,
+  fromName,
+  to,
+  subject,
+  content,
+  cc,
+  bcc,
+  attachments
+}: Props): Promise<Response> => {
+  try {
+    // 验证SMTP配置
+    if (!smtpHost || !smtpPort || !smtpUser || !smtpPass) {
+      throw new Error('Incomplete SMTP configuration');
+    }
+
+    // 验证必填参数
+    if (!to || !subject || !content) {
+      throw new Error('Recipient, subject, and content are required');
+    }
+
+    // 验证邮箱格式
+    if (!validateEmails(to)) {
+      throw new Error('Invalid recipient email format');
+    }
+    if (cc && !validateEmails(cc)) {
+      throw new Error('Invalid CC email format');
+    }
+    if (bcc && !validateEmails(bcc)) {
+      throw new Error('Invalid BCC email format');
+    }
+
+    // 创建SMTP传输对象
+    const transporter = nodemailer.createTransport({
+      host: smtpHost,
+      port: Number(smtpPort),
+      secure: SSL === true,
+      auth: {
+        user: smtpUser,
+        pass: smtpPass
+      }
+    });
+
+    let attachmentsArray = [];
+    try {
+      attachmentsArray = JSON.parse(attachments || '[]');
+    } catch (error) {
+      throw new Error('Attachment format parsing error, please check attachment configuration');
+    }
+
+    // 发送邮件
+    const info = await transporter.sendMail({
+      from: `"${fromName || 'FastGPT'}" <${smtpUser}>`,
+      to: to
+        .split(',')
+        .map((email) => email.trim())
+        .join(','),
+      cc: cc
+        ?.split(',')
+        .map((email) => email.trim())
+        .join(','),
+      bcc: bcc
+        ?.split(',')
+        .map((email) => email.trim())
+        .join(','),
+      subject,
+      html: content,
+      attachments: attachmentsArray || []
+    });
+
+    return {
+      success: true,
+      messageId: info.messageId
+    };
+  } catch (error: any) {
+    return {
+      success: false,
+      error: getErrText(error)
+    };
+  }
+};
+
+export default main;

packages/plugins/src/smtpEmail/template.json

@@ -0,0 +1,651 @@
+{
+  "author": "cloudpense",
+  "version": "1.0.0",
+  "name": "Email 邮件发送",
+  "avatar": "plugins/email",
+  "intro": "通过SMTP协议发送电子邮件(nodemailer)",
+  "showStatus": true,
+  "weight": 10,
+  "isTool": true,
+  "templateType": "tools",
+
+  "workflow": {
+    "nodes": [
+      {
+        "nodeId": "pluginInput",
+        "name": "workflow:template.plugin_start",
+        "intro": "workflow:intro_plugin_input",
+        "avatar": "core/workflow/template/workflowStart",
+        "flowNodeType": "pluginInput",
+        "showStatus": false,
+        "position": {
+          "x": 595.3456736313964,
+          "y": -323.02524442647456
+        },
+        "version": "481",
+        "inputs": [
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "smtpHost",
+            "label": "smtpHost",
+            "description": "",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": true,
+            "customInputConfig": {
+              "selectValueTypeList": ["string"]
+            }
+          },
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "smtpPort",
+            "label": "smtpPort",
+            "description": "SMTP端口",
+            "defaultValue": "465",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": true
+          },
+          {
+            "renderTypeList": ["select", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "SSL",
+            "label": "SSL",
+            "description": "SSL",
+            "defaultValue": "true",
+            "list": [
+              {
+                "label": "true",
+                "value": "true"
+              },
+              {
+                "label": "false",
+                "value": "false"
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": true
+          },
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "smtpUser",
+            "label": "smtpUser",
+            "description": "SMTP用户名, 邮箱账号",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": true
+          },
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "smtpPass",
+            "label": "smtpPass",
+            "description": "邮箱密码或授权码",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": true
+          },
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "fromName",
+            "label": "fromName",
+            "description": "显示的发件人名称",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": true
+          },
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "to",
+            "label": "to",
+            "description": "请输入收件人邮箱，多个邮箱用逗号分隔",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": true,
+            "toolDescription": "请输入收件人邮箱，多个邮箱用逗号分隔"
+          },
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "subject",
+            "label": "subject",
+            "description": "请输入邮件主题",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": true,
+            "toolDescription": "请输入邮件主题"
+          },
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "content",
+            "label": "content",
+            "description": "请输入邮件内容，支持HTML格式",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": true,
+            "toolDescription": "请输入邮件内容，支持HTML格式"
+          },
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "cc",
+            "label": "cc",
+            "description": "请输入抄送邮箱，多个邮箱用逗号分隔",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": false,
+            "toolDescription": "请输入抄送邮箱，多个邮箱用逗号分隔"
+          },
+          {
+            "renderTypeList": ["input", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "bcc",
+            "label": "bcc",
+            "description": "请输入密送邮箱，多个邮箱用逗号分隔",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": false,
+            "toolDescription": "请输入密送邮箱，多个邮箱用逗号分隔"
+          },
+          {
+            "renderTypeList": ["JSONEditor", "reference"],
+            "selectedTypeIndex": 0,
+            "valueType": "string",
+            "canEdit": true,
+            "key": "attachments",
+            "label": "attachments",
+            "description": "必须是json数组格式\n[{\"filename\":\"附件名\",\"path\":\"附件url\"}]",
+            "defaultValue": "",
+            "list": [
+              {
+                "label": "",
+                "value": ""
+              }
+            ],
+            "maxFiles": 5,
+            "canSelectFile": true,
+            "canSelectImg": true,
+            "required": false,
+            "customInputConfig": {
+              "selectValueTypeList": ["arrayObject"]
+            },
+            "toolDescription": "必须是json数组格式\n[{\"filename\":\"附件名\",\"path\":\"附件url\"}]",
+            "maxLength": 0
+          }
+        ],
+        "outputs": [
+          {
+            "id": "smtpHost",
+            "valueType": "string",
+            "key": "smtpHost",
+            "label": "smtpHost",
+            "type": "hidden"
+          },
+          {
+            "id": "smtpPort",
+            "valueType": "string",
+            "key": "smtpPort",
+            "label": "smtpPort",
+            "type": "hidden"
+          },
+          {
+            "id": "SSL",
+            "valueType": "string",
+            "key": "SSL",
+            "label": "SSL",
+            "type": "hidden"
+          },
+          {
+            "id": "smtpUser",
+            "valueType": "string",
+            "key": "smtpUser",
+            "label": "smtpUser",
+            "type": "hidden"
+          },
+          {
+            "id": "smtpPass",
+            "valueType": "string",
+            "key": "smtpPass",
+            "label": "smtpPass",
+            "type": "hidden"
+          },
+          {
+            "id": "fromName",
+            "valueType": "string",
+            "key": "fromName",
+            "label": "fromName",
+            "type": "hidden"
+          },
+          {
+            "id": "to",
+            "valueType": "string",
+            "key": "to",
+            "label": "to",
+            "type": "hidden"
+          },
+          {
+            "id": "subject",
+            "valueType": "string",
+            "key": "subject",
+            "label": "subject",
+            "type": "hidden"
+          },
+          {
+            "id": "content",
+            "valueType": "string",
+            "key": "content",
+            "label": "content",
+            "type": "hidden"
+          },
+          {
+            "id": "cc",
+            "valueType": "string",
+            "key": "cc",
+            "label": "cc",
+            "type": "hidden"
+          },
+          {
+            "id": "bcc",
+            "valueType": "string",
+            "key": "bcc",
+            "label": "bcc",
+            "type": "hidden"
+          },
+          {
+            "id": "attachments",
+            "valueType": "string",
+            "key": "attachments",
+            "label": "attachments",
+            "type": "hidden"
+          }
+        ]
+      },
+      {
+        "nodeId": "pluginOutput",
+        "name": "common:core.module.template.self_output",
+        "intro": "workflow:intro_custom_plugin_output",
+        "avatar": "core/workflow/template/pluginOutput",
+        "flowNodeType": "pluginOutput",
+        "showStatus": false,
+        "position": {
+          "x": 2135.4991928806685,
+          "y": -98.02524442647456
+        },
+        "version": "481",
+        "inputs": [
+          {
+            "renderTypeList": ["reference"],
+            "valueType": "string",
+            "canEdit": true,
+            "key": "发送结果",
+            "label": "发送结果",
+            "isToolOutput": true,
+            "description": "",
+            "required": true,
+            "value": ["uOX6ITvPWm9O", "httpRawResponse"]
+          }
+        ],
+        "outputs": []
+      },
+      {
+        "nodeId": "pluginConfig",
+        "name": "common:core.module.template.system_config",
+        "intro": "",
+        "avatar": "core/workflow/template/systemConfig",
+        "flowNodeType": "pluginConfig",
+        "position": {
+          "x": 184.66337662472682,
+          "y": -216.05298493910115
+        },
+        "version": "4811",
+        "inputs": [],
+        "outputs": []
+      },
+      {
+        "nodeId": "uOX6ITvPWm9O",
+        "name": "HTTP 请求",
+        "intro": "可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）",
+        "avatar": "core/workflow/template/httpRequest",
+        "flowNodeType": "httpRequest468",
+        "showStatus": true,
+        "position": {
+          "x": 1340.0519095857342,
+          "y": -393.02524442647456
+        },
+        "version": "481",
+        "inputs": [
+          {
+            "key": "system_addInputParam",
+            "renderTypeList": ["addInputParam"],
+            "valueType": "dynamic",
+            "label": "",
+            "required": false,
+            "description": "common:core.module.input.description.HTTP Dynamic Input",
+            "customInputConfig": {
+              "selectValueTypeList": [
+                "string",
+                "number",
+                "boolean",
+                "object",
+                "arrayString",
+                "arrayNumber",
+                "arrayBoolean",
+                "arrayObject",
+                "arrayAny",
+                "any",
+                "chatHistory",
+                "datasetQuote",
+                "dynamic",
+                "selectDataset",
+                "selectApp"
+              ],
+              "showDescription": false,
+              "showDefaultValue": true
+            },
+            "valueDesc": "",
+            "debugLabel": "",
+            "toolDescription": ""
+          },
+          {
+            "key": "system_httpMethod",
+            "renderTypeList": ["custom"],
+            "valueType": "string",
+            "label": "",
+            "value": "POST",
+            "required": true,
+            "valueDesc": "",
+            "description": "",
+            "debugLabel": "",
+            "toolDescription": ""
+          },
+          {
+            "key": "system_httpTimeout",
+            "renderTypeList": ["custom"],
+            "valueType": "number",
+            "label": "",
+            "value": 30,
+            "min": 5,
+            "max": 600,
+            "required": true,
+            "valueDesc": "",
+            "description": "",
+            "debugLabel": "",
+            "toolDescription": ""
+          },
+          {
+            "key": "system_httpReqUrl",
+            "renderTypeList": ["hidden"],
+            "valueType": "string",
+            "label": "",
+            "description": "common:core.module.input.description.Http Request Url",
+            "placeholder": "https://api.ai.com/getInventory",
+            "required": false,
+            "value": "smtpEmail",
+            "valueDesc": "",
+            "debugLabel": "",
+            "toolDescription": ""
+          },
+          {
+            "key": "system_httpHeader",
+            "renderTypeList": ["custom"],
+            "valueType": "any",
+            "value": [],
+            "label": "",
+            "description": "common:core.module.input.description.Http Request Header",
+            "placeholder": "common:core.module.input.description.Http Request Header",
+            "required": false,
+            "valueDesc": "",
+            "debugLabel": "",
+            "toolDescription": ""
+          },
+          {
+            "key": "system_httpParams",
+            "renderTypeList": ["hidden"],
+            "valueType": "any",
+            "value": [],
+            "label": "",
+            "required": false,
+            "valueDesc": "",
+            "description": "",
+            "debugLabel": "",
+            "toolDescription": ""
+          },
+          {
+            "key": "system_httpJsonBody",
+            "renderTypeList": ["hidden"],
+            "valueType": "any",
+            "value": "{\n\"smtpHost\": \"{{$pluginInput.smtpHost$}}\",\n\"smtpPort\": \"{{$pluginInput.smtpPort$}}\",\n\"SSL\": {{$pluginInput.SSL$}},\n\"smtpUser\": \"{{$pluginInput.smtpUser$}}\",\n\"smtpPass\": \"{{$pluginInput.smtpPass$}}\",\n\"fromName\": \"{{$pluginInput.fromName$}}\",\n\"to\": \"{{$pluginInput.to$}}\",\n\"subject\": \"{{$pluginInput.subject$}}\",\n\"content\": \"{{$pluginInput.content$}}\",\n\"cc\": \"{{$pluginInput.cc$}}\",\n\"bcc\": \"{{$pluginInput.bcc$}}\",\n\"attachments\":'{{$pluginInput.attachments$}}'\n}",
+            "label": "",
+            "required": false,
+            "valueDesc": "",
+            "description": "",
+            "debugLabel": "",
+            "toolDescription": ""
+          },
+          {
+            "key": "system_httpFormBody",
+            "renderTypeList": ["hidden"],
+            "valueType": "any",
+            "value": [],
+            "label": "",
+            "required": false,
+            "valueDesc": "",
+            "description": "",
+            "debugLabel": "",
+            "toolDescription": ""
+          },
+          {
+            "key": "system_httpContentType",
+            "renderTypeList": ["hidden"],
+            "valueType": "string",
+            "value": "json",
+            "label": "",
+            "required": false,
+            "debugLabel": "",
+            "toolDescription": ""
+          }
+        ],
+        "outputs": [
+          {
+            "id": "error",
+            "key": "error",
+            "label": "workflow:request_error",
+            "description": "HTTP请求错误信息，成功时返回空",
+            "valueType": "object",
+            "type": "static"
+          },
+          {
+            "id": "httpRawResponse",
+            "key": "httpRawResponse",
+            "required": true,
+            "label": "workflow:raw_response",
+            "description": "HTTP请求的原始响应。只能接受字符串或JSON类型响应数据。",
+            "valueType": "any",
+            "type": "static"
+          },
+          {
+            "id": "system_addOutputParam",
+            "key": "system_addOutputParam",
+            "type": "dynamic",
+            "valueType": "dynamic",
+            "label": "输出字段提取",
+            "customFieldConfig": {
+              "selectValueTypeList": [
+                "string",
+                "number",
+                "boolean",
+                "object",
+                "arrayString",
+                "arrayNumber",
+                "arrayBoolean",
+                "arrayObject",
+                "arrayAny",
+                "any",
+                "chatHistory",
+                "datasetQuote",
+                "dynamic",
+                "selectApp",
+                "selectDataset"
+              ],
+              "showDescription": false,
+              "showDefaultValue": false
+            },
+            "description": "可以通过 JSONPath 语法来提取响应值中的指定字段",
+            "valueDesc": ""
+          }
+        ]
+      }
+    ],
+    "edges": [
+      {
+        "source": "uOX6ITvPWm9O",
+        "target": "pluginOutput",
+        "sourceHandle": "uOX6ITvPWm9O-source-right",
+        "targetHandle": "pluginOutput-target-left"
+      },
+      {
+        "source": "pluginInput",
+        "target": "uOX6ITvPWm9O",
+        "sourceHandle": "pluginInput-source-right",
+        "targetHandle": "uOX6ITvPWm9O-target-left"
+      }
+    ],
+    "chatConfig": {
+      "welcomeText": "",
+      "variables": [],
+      "questionGuide": {
+        "open": false,
+        "model": "gpt-4o-mini",
+        "customPrompt": ""
+      },
+      "ttsConfig": {
+        "type": "web"
+      },
+      "whisperConfig": {
+        "open": false,
+        "autoSend": false,
+        "autoTTSResponse": false
+      },
+      "chatInputGuide": {
+        "open": false,
+        "textList": [],
+        "customUrl": ""
+      },
+      "instruction": "通过SMTP协议发送电子邮件",
+      "autoExecute": {
+        "open": false,
+        "defaultPrompt": ""
+      },
+      "_id": "67ad649ea4b6b8eefa9d3d0d"
+    }
+  }
+}

packages/service/common/api/type.d.ts

@@ -0,0 +1,29 @@
+import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
+import {
+  DeepRagSearchProps,
+  SearchDatasetDataResponse
+} from '../../core/dataset/search/controller';
+import { AuthOpenApiLimitProps } from '../../support/openapi/auth';
+import { CreateUsageProps, ConcatUsageProps } from '@fastgpt/global/support/wallet/usage/api';
+import {
+  GetProApiDatasetFileContentParams,
+  GetProApiDatasetFileListParams,
+  GetProApiDatasetFilePreviewUrlParams
+} from '../../core/dataset/apiDataset/proApi';
+
+declare global {
+  var textCensorHandler: (params: { text: string }) => Promise<{ code: number; message?: string }>;
+  var deepRagHandler: (data: DeepRagSearchProps) => Promise<SearchDatasetDataResponse>;
+  var authOpenApiHandler: (data: AuthOpenApiLimitProps) => Promise<any>;
+  var createUsageHandler: (data: CreateUsageProps) => Promise<void>;
+  var concatUsageHandler: (data: ConcatUsageProps) => Promise<void>;
+
+  // API dataset
+  var getProApiDatasetFileList: (data: GetProApiDatasetFileListParams) => Promise<APIFileItem[]>;
+  var getProApiDatasetFileContent: (
+    data: GetProApiDatasetFileContentParams
+  ) => Promise<ApiFileReadContentResponse>;
+  var getProApiDatasetFilePreviewUrl: (
+    data: GetProApiDatasetFilePreviewUrlParams
+  ) => Promise<string>;
+}

packages/service/common/buffer/rawText/schema.ts

@@ -1,5 +1,4 @@
-import { connectionMongo, getMongoModel } from '../../mongo';
+import { getMongoModel, Schema } from '../../mongo';
-const { Schema } = connectionMongo;
 import { RawTextBufferSchemaType } from './type';
 
 export const collectionName = 'buffer_rawtexts';

packages/service/common/buffer/tts/schema.ts

@@ -1,5 +1,4 @@
-import { connectionMongo, getMongoModel, type Model } from '../../../common/mongo';
+import { Schema, getMongoModel } from '../../../common/mongo';
-const { Schema, model, models } = connectionMongo;
 import { TTSBufferSchemaType } from './type.d';
 
 export const collectionName = 'buffer_tts';

packages/service/common/bullmq/index.ts

@@ -0,0 +1,79 @@
+import { ConnectionOptions, Processor, Queue, QueueOptions, Worker, WorkerOptions } from 'bullmq';
+import { addLog } from '../system/log';
+import { newQueueRedisConnection, newWorkerRedisConnection } from '../redis';
+
+const defaultWorkerOpts: Omit<ConnectionOptions, 'connection'> = {
+  removeOnComplete: {
+    count: 0 // Delete jobs immediately on completion
+  },
+  removeOnFail: {
+    count: 0 // Delete jobs immediately on failure
+  }
+};
+
+export enum QueueNames {
+  websiteSync = 'websiteSync'
+}
+
+export const queues = (() => {
+  if (!global.queues) {
+    global.queues = new Map<QueueNames, Queue>();
+  }
+  return global.queues;
+})();
+export const workers = (() => {
+  if (!global.workers) {
+    global.workers = new Map<QueueNames, Worker>();
+  }
+  return global.workers;
+})();
+
+export function getQueue<DataType, ReturnType = void>(
+  name: QueueNames,
+  opts?: Omit<QueueOptions, 'connection'>
+): Queue<DataType, ReturnType> {
+  // check if global.queues has the queue
+  const queue = queues.get(name);
+  if (queue) {
+    return queue as Queue<DataType, ReturnType>;
+  }
+  const newQueue = new Queue<DataType, ReturnType>(name.toString(), {
+    connection: newQueueRedisConnection(),
+    ...opts
+  });
+
+  // default error handler, to avoid unhandled exceptions
+  newQueue.on('error', (error) => {
+    addLog.error(`MQ Queue [${name}]: ${error.message}`, error);
+  });
+  queues.set(name, newQueue);
+  return newQueue;
+}
+
+export function getWorker<DataType, ReturnType = void>(
+  name: QueueNames,
+  processor: Processor<DataType, ReturnType>,
+  opts?: Omit<WorkerOptions, 'connection'>
+): Worker<DataType, ReturnType> {
+  const worker = workers.get(name);
+  if (worker) {
+    return worker as Worker<DataType, ReturnType>;
+  }
+
+  const newWorker = new Worker<DataType, ReturnType>(name.toString(), processor, {
+    connection: newWorkerRedisConnection(),
+    ...defaultWorkerOpts,
+    ...opts
+  });
+  // default error handler, to avoid unhandled exceptions
+  newWorker.on('error', (error) => {
+    addLog.error(`MQ Worker [${name}]: ${error.message}`, error);
+  });
+  newWorker.on('failed', (jobId, error) => {
+    addLog.error(`MQ Worker [${name}]: ${error.message}`, error);
+  });
+  workers.set(name, newWorker);
+  return newWorker;
+}
+
+export * from 'bullmq';

packages/service/common/bullmq/type.d.ts

@@ -0,0 +1,7 @@
+import { Queue, Worker } from 'bullmq';
+import { QueueNames } from './index';
+
+declare global {
+  var queues: Map<QueueNames, Queue> | undefined;
+  var workers: Map<QueueNames, Worker> | undefined;
+}

packages/service/common/file/gridfs/schema.ts

@@ -1,5 +1,4 @@
-import { connectionMongo, getMongoModel, type Model } from '../../mongo';
+import { Schema, getMongoModel } from '../../mongo';
-const { Schema } = connectionMongo;
 
 const DatasetFileSchema = new Schema({});
 const ChatFileSchema = new Schema({});

packages/service/common/file/image/schema.ts

@@ -1,7 +1,6 @@
 import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
-import { connectionMongo, getMongoModel } from '../../mongo';
+import { Schema, getMongoModel } from '../../mongo';
 import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d';
-const { Schema } = connectionMongo;
 
 const ImageSchema = new Schema({
   teamId: {

packages/service/common/mongo/index.ts

@@ -1,17 +1,26 @@
 import { addLog } from '../../common/system/log';
-import mongoose, { Model } from 'mongoose';
+import mongoose, { Model, Mongoose } from 'mongoose';
 
 export default mongoose;
 export * from 'mongoose';
 
+export const MONGO_URL = process.env.MONGODB_URI as string;
+export const MONGO_LOG_URL = (process.env.MONGODB_LOG_URI ?? process.env.MONGODB_URI) as string;
+
 export const connectionMongo = (() => {
   if (!global.mongodb) {
-    global.mongodb = mongoose;
+    global.mongodb = new Mongoose();
   }
-
   return global.mongodb;
 })();
 
+export const connectionLogMongo = (() => {
+  if (!global.mongodbLog) {
+    global.mongodbLog = new Mongoose();
+  }
+  return global.mongodbLog;
+})();
+
 const addCommonMiddleware = (schema: mongoose.Schema) => {
   const operations = [
     /^find/,
@@ -60,7 +69,7 @@ const addCommonMiddleware = (schema: mongoose.Schema) => {
 
 export const getMongoModel = <T>(name: string, schema: mongoose.Schema) => {
   if (connectionMongo.models[name]) return connectionMongo.models[name] as Model<T>;
-  console.log('Load model======', name);
+  if (process.env.NODE_ENV !== 'test') console.log('Load model======', name);
   addCommonMiddleware(schema);
 
   const model = connectionMongo.model<T>(name, schema);
@@ -71,6 +80,19 @@ export const getMongoModel = <T>(name: string, schema: mongoose.Schema) => {
   return model;
 };
 
+export const getMongoLogModel = <T>(name: string, schema: mongoose.Schema) => {
+  if (connectionLogMongo.models[name]) return connectionLogMongo.models[name] as Model<T>;
+  console.log('Load model======', name);
+  addCommonMiddleware(schema);
+
+  const model = connectionLogMongo.model<T>(name, schema);
+
+  // Sync index
+  syncMongoIndex(model);
+
+  return model;
+};
+
 const syncMongoIndex = async (model: Model<any>) => {
   if (process.env.SYNC_INDEX !== '0' && process.env.NODE_ENV !== 'test') {
     try {

packages/service/common/mongo/init.ts

@@ -1,6 +1,5 @@
 import { delay } from '@fastgpt/global/common/system/utils';
 import { addLog } from '../system/log';
-import { connectionMongo } from './index';
 import type { Mongoose } from 'mongoose';
 
 const maxConnecting = Math.max(30, Number(process.env.DB_MAX_LINK || 20));
@@ -8,41 +7,41 @@ const maxConnecting = Math.max(30, Number(process.env.DB_MAX_LINK || 20));
 /**
  * connect MongoDB and init data
  */
-export async function connectMongo(): Promise<Mongoose> {
+export async function connectMongo(db: Mongoose, url: string): Promise<Mongoose> {
   /* Connecting, connected will return */
-  if (connectionMongo.connection.readyState !== 0) {
+  if (db.connection.readyState !== 0) {
-    return connectionMongo;
+    return db;
   }
 
-  console.log('mongo start connect');
+  console.log('MongoDB start connect');
   try {
     // Remove existing listeners to prevent duplicates
-    connectionMongo.connection.removeAllListeners('error');
+    db.connection.removeAllListeners('error');
-    connectionMongo.connection.removeAllListeners('disconnected');
+    db.connection.removeAllListeners('disconnected');
-    connectionMongo.set('strictQuery', 'throw');
+    db.set('strictQuery', 'throw');
 
-    connectionMongo.connection.on('error', async (error) => {
+    db.connection.on('error', async (error) => {
       console.log('mongo error', error);
       try {
-        if (connectionMongo.connection.readyState !== 0) {
+        if (db.connection.readyState !== 0) {
-          await connectionMongo.disconnect();
+          await db.disconnect();
           await delay(1000);
-          await connectMongo();
+          await connectMongo(db, url);
         }
       } catch (error) {}
     });
-    connectionMongo.connection.on('disconnected', async () => {
+    db.connection.on('disconnected', async () => {
       console.log('mongo disconnected');
       try {
-        if (connectionMongo.connection.readyState !== 0) {
+        if (db.connection.readyState !== 0) {
-          await connectionMongo.disconnect();
+          await db.disconnect();
           await delay(1000);
-          await connectMongo();
+          await connectMongo(db, url);
         }
       } catch (error) {}
     });
 
-    await connectionMongo.connect(process.env.MONGODB_URI as string, {
+    const options = {
       bufferCommands: true,
       maxConnecting: maxConnecting,
       maxPoolSize: maxConnecting,
@@ -53,18 +52,18 @@ export async function connectMongo(): Promise<Mongoose> {
       maxIdleTimeMS: 300000,
       retryWrites: true,
       retryReads: true
+    };
 
-      // readPreference: 'secondaryPreferred',
+    db.connect(url, options);
-      // readConcern: { level: 'local' },
-      // writeConcern: { w: 'majority', j: true }
-    });
 
     console.log('mongo connected');
-    return connectionMongo;
+    return db;
   } catch (error) {
-    addLog.error('mongo connect error', error);
+    addLog.error('Mongo connect error', error);
-    await connectionMongo.disconnect();
+
+    await db.disconnect();
+
     await delay(1000);
-    return connectMongo();
+    return connectMongo(db, url);
   }
 }

packages/service/common/mongo/type.d.ts

@@ -3,4 +3,5 @@ import type { Logger } from 'winston';
 
 declare global {
   var mongodb: Mongoose | undefined;
+  var mongodbLog: Mongoose | undefined;
 }

packages/service/common/redis/cache.ts

@@ -0,0 +1,38 @@
+import { getGlobalRedisCacheConnection } from './index';
+import { addLog } from '../system/log';
+import { retryFn } from '@fastgpt/global/common/system/utils';
+
+export enum CacheKeyEnum {
+  team_vector_count = 'team_vector_count'
+}
+
+export const setRedisCache = async (
+  key: string,
+  data: string | Buffer | number,
+  expireSeconds?: number
+) => {
+  return await retryFn(async () => {
+    try {
+      const redis = getGlobalRedisCacheConnection();
+
+      if (expireSeconds) {
+        await redis.set(key, data, 'EX', expireSeconds);
+      } else {
+        await redis.set(key, data);
+      }
+    } catch (error) {
+      addLog.error('Set cache error:', error);
+      return Promise.reject(error);
+    }
+  });
+};
+
+export const getRedisCache = async (key: string) => {
+  const redis = getGlobalRedisCacheConnection();
+  return await retryFn(() => redis.get(key));
+};
+
+export const delRedisCache = async (key: string) => {
+  const redis = getGlobalRedisCacheConnection();
+  await retryFn(() => redis.del(key));
+};

packages/service/common/redis/index.ts

@@ -0,0 +1,43 @@
+import { addLog } from '../system/log';
+import Redis from 'ioredis';
+
+const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379';
+
+export const newQueueRedisConnection = () => {
+  const redis = new Redis(REDIS_URL);
+  redis.on('connect', () => {
+    console.log('Redis connected');
+  });
+  redis.on('error', (error) => {
+    console.error('Redis connection error', error);
+  });
+  return redis;
+};
+
+export const newWorkerRedisConnection = () => {
+  const redis = new Redis(REDIS_URL, {
+    maxRetriesPerRequest: null
+  });
+  redis.on('connect', () => {
+    console.log('Redis connected');
+  });
+  redis.on('error', (error) => {
+    console.error('Redis connection error', error);
+  });
+  return redis;
+};
+
+export const getGlobalRedisCacheConnection = () => {
+  if (global.redisCache) return global.redisCache;
+
+  global.redisCache = new Redis(REDIS_URL, { keyPrefix: 'fastgpt:cache:' });
+
+  global.redisCache.on('connect', () => {
+    addLog.info('Redis connected');
+  });
+  global.redisCache.on('error', (error) => {
+    addLog.error('Redis connection error', error);
+  });
+
+  return global.redisCache;
+};

packages/service/common/redis/type.d.ts

@@ -0,0 +1,5 @@
+import Redis from 'ioredis';
+
+declare global {
+  var redisCache: Redis | null;
+}

packages/service/common/system/constants.ts

@@ -1,4 +1,3 @@
 export const FastGPTProUrl = process.env.PRO_URL ? `${process.env.PRO_URL}/api` : '';
-export const isFastGPTMainService = !!process.env.PRO_URL;
 // @ts-ignore
 export const isFastGPTProService = () => !!global.systemConfig;

packages/service/common/system/log/schema.ts

@@ -1,4 +1,4 @@
-import { getMongoModel, Schema } from '../../../common/mongo';
+import { getMongoLogModel as getMongoModel, Schema } from '../../../common/mongo';
 import { SystemLogType } from './type';
 import { LogLevelEnum } from './constant';
 

packages/service/common/system/timerLock/constants.ts

@@ -1,5 +1,5 @@
 export enum TimerIdEnum {
-  checkInValidDatasetFiles = 'checkInValidDatasetFiles',
+  checkExpiredFiles = 'checkExpiredFiles',
   checkInvalidDatasetData = 'checkInvalidDatasetData',
   checkInvalidVector = 'checkInvalidVector',
   clearExpiredSubPlan = 'clearExpiredSubPlan',

packages/service/common/vectorStore/controller.ts

@@ -2,10 +2,13 @@
 import { PgVectorCtrl } from './pg/class';
 import { ObVectorCtrl } from './oceanbase/class';
 import { getVectorsByText } from '../../core/ai/embedding';
-import { InsertVectorProps } from './controller.d';
+import { DelDatasetVectorCtrlProps, InsertVectorProps } from './controller.d';
 import { EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { MILVUS_ADDRESS, PG_ADDRESS, OCEANBASE_ADDRESS } from './constants';
 import { MilvusCtrl } from './milvus/class';
+import { setRedisCache, getRedisCache, delRedisCache, CacheKeyEnum } from '../redis/cache';
+import { throttle } from 'lodash';
+import { retryFn } from '@fastgpt/global/common/system/utils';
 
 const getVectorObj = () => {
   if (PG_ADDRESS) return new PgVectorCtrl();
@@ -15,13 +18,33 @@ const getVectorObj = () => {
   return new PgVectorCtrl();
 };
 
+const getChcheKey = (teamId: string) => `${CacheKeyEnum.team_vector_count}:${teamId}`;
+const onDelCache = throttle((teamId: string) => delRedisCache(getChcheKey(teamId)), 30000, {
+  leading: true,
+  trailing: true
+});
+
 const Vector = getVectorObj();
 
 export const initVectorStore = Vector.init;
-export const deleteDatasetDataVector = Vector.delete;
 export const recallFromVectorStore = Vector.embRecall;
 export const getVectorDataByTime = Vector.getVectorDataByTime;
-export const getVectorCountByTeamId = Vector.getVectorCountByTeamId;
+
+export const getVectorCountByTeamId = async (teamId: string) => {
+  const key = getChcheKey(teamId);
+
+  const countStr = await getRedisCache(key);
+  if (countStr) {
+    return Number(countStr);
+  }
+
+  const count = await Vector.getVectorCountByTeamId(teamId);
+
+  await setRedisCache(key, count, 30 * 60);
+
+  return count;
+};
+
 export const getVectorCountByDatasetId = Vector.getVectorCountByDatasetId;
 export const getVectorCountByCollectionId = Vector.getVectorCountByCollectionId;
 
@@ -33,18 +56,28 @@ export const insertDatasetDataVector = async ({
   query: string;
   model: EmbeddingModelItemType;
 }) => {
-  const { vectors, tokens } = await getVectorsByText({
+  return retryFn(async () => {
-    model,
+    const { vectors, tokens } = await getVectorsByText({
-    input: query,
+      model,
-    type: 'db'
+      input: query,
-  });
+      type: 'db'
-  const { insertId } = await Vector.insert({
+    });
-    ...props,
+    const { insertId } = await Vector.insert({
-    vector: vectors[0]
+      ...props,
-  });
+      vector: vectors[0]
+    });
 
-  return {
+    onDelCache(props.teamId);
-    tokens,
+
-    insertId
+    return {
-  };
+      tokens,
+      insertId
+    };
+  });
+};
+
+export const deleteDatasetDataVector = async (props: DelDatasetVectorCtrlProps) => {
+  const result = await Vector.delete(props);
+  onDelCache(props.teamId);
+  return result;
 };

packages/service/core/ai/utils.ts

@@ -2,6 +2,7 @@ import { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
 import {
   ChatCompletionCreateParamsNonStreaming,
   ChatCompletionCreateParamsStreaming,
+  CompletionFinishReason,
   StreamChatType
 } from '@fastgpt/global/core/ai/type';
 import { getLLMModel } from './model';
@@ -142,26 +143,40 @@ export const parseReasoningStreamContent = () => {
           content?: string;
           reasoning_content?: string;
         };
+        finish_reason?: CompletionFinishReason;
       }[];
     },
     parseThinkTag = false
-  ): [string, string] => {
+  ): {
+    reasoningContent: string;
+    content: string;
+    finishReason: CompletionFinishReason;
+  } => {
     const content = part.choices?.[0]?.delta?.content || '';
+    const finishReason = part.choices?.[0]?.finish_reason || null;
 
     // @ts-ignore
     const reasoningContent = part.choices?.[0]?.delta?.reasoning_content || '';
     if (reasoningContent || !parseThinkTag) {
       isInThinkTag = false;
-      return [reasoningContent, content];
+      return { reasoningContent, content, finishReason };
     }
 
     if (!content) {
-      return ['', ''];
+      return {
+        reasoningContent: '',
+        content: '',
+        finishReason
+      };
     }
 
     // 如果不在 think 标签中，或者有 reasoningContent(接口已解析），则返回 reasoningContent 和 content
     if (isInThinkTag === false) {
-      return ['', content];
+      return {
+        reasoningContent: '',
+        content,
+        finishReason
+      };
     }
 
     // 检测是否为 think 标签开头的数据
@@ -170,17 +185,29 @@ export const parseReasoningStreamContent = () => {
       startTagBuffer += content;
       // 太少内容时候，暂时不解析
       if (startTagBuffer.length < startTag.length) {
-        return ['', ''];
+        return {
+          reasoningContent: '',
+          content: '',
+          finishReason
+        };
       }
 
       if (startTagBuffer.startsWith(startTag)) {
         isInThinkTag = true;
-        return [startTagBuffer.slice(startTag.length), ''];
+        return {
+          reasoningContent: startTagBuffer.slice(startTag.length),
+          content: '',
+          finishReason
+        };
       }
 
       // 如果未命中 think 标签，则认为不在 think 标签中，返回 buffer 内容作为 content
       isInThinkTag = false;
-      return ['', startTagBuffer];
+      return {
+        reasoningContent: '',
+        content: startTagBuffer,
+        finishReason
+      };
     }
 
     // 确认是 think 标签内容，开始返回 think 内容，并实时检测 </think>
@@ -201,19 +228,35 @@ export const parseReasoningStreamContent = () => {
       if (endTagBuffer.includes(endTag)) {
         isInThinkTag = false;
         const answer = endTagBuffer.slice(endTag.length);
-        return ['', answer];
+        return {
+          reasoningContent: '',
+          content: answer,
+          finishReason
+        };
       } else if (endTagBuffer.length >= endTag.length) {
         // 缓存内容超出尾标签长度，且仍未命中 </think>，则认为本次猜测 </think> 失败，仍处于 think 阶段。
         const tmp = endTagBuffer;
         endTagBuffer = '';
-        return [tmp, ''];
+        return {
+          reasoningContent: tmp,
+          content: '',
+          finishReason
+        };
       }
-      return ['', ''];
+      return {
+        reasoningContent: '',
+        content: '',
+        finishReason
+      };
     } else if (content.includes(endTag)) {
       // 返回内容，完整命中</think>，直接结束
       isInThinkTag = false;
       const [think, answer] = content.split(endTag);
-      return [think, answer];
+      return {
+        reasoningContent: think,
+        content: answer,
+        finishReason
+      };
     } else {
       // 无 buffer，且未命中 </think>，开始疑似 </think> 检测。
       for (let i = 1; i < endTag.length; i++) {
@@ -222,13 +265,21 @@ export const parseReasoningStreamContent = () => {
         if (content.endsWith(partialEndTag)) {
           const think = content.slice(0, -partialEndTag.length);
           endTagBuffer += partialEndTag;
-          return [think, ''];
+          return {
+            reasoningContent: think,
+            content: '',
+            finishReason
+          };
         }
       }
     }
 
     // 完全未命中尾标签，还是 think 阶段。
-    return [content, ''];
+    return {
+      reasoningContent: content,
+      content: '',
+      finishReason
+    };
   };
 
   const getStartTagBuffer = () => startTagBuffer;

packages/service/core/chat/postTextCensor.ts

@@ -1,7 +1,6 @@
-import { POST } from './plusRequest';
-
 export const postTextCensor = (data: { text: string }) =>
-  POST<{ code?: number; message: string }>('/common/censor/check', data)
+  global
+    .textCensorHandler(data)
     .then((res) => {
       if (res?.code === 5000) {
         return Promise.reject(res);

packages/service/core/chat/saveChat.ts

@@ -16,6 +16,7 @@ import { mergeChatResponseData } from '@fastgpt/global/core/chat/utils';
 import { pushChatLog } from './pushChatLog';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
+import { extractDeepestInteractive } from '@fastgpt/global/core/workflow/runtime/utils';
 
 type Props = {
   chatId: string;
@@ -209,34 +210,24 @@ export const updateInteractiveChat = async ({
     }
   })();
 
-  if (interactiveValue.interactive.type === 'userSelect') {
+  let finalInteractive = extractDeepestInteractive(interactiveValue.interactive);
-    interactiveValue.interactive = {
+
-      ...interactiveValue.interactive,
+  if (finalInteractive.type === 'userSelect') {
-      params: {
+    finalInteractive.params.userSelectedVal = userInteractiveVal;
-        ...interactiveValue.interactive.params,
-        userSelectedVal: userInteractiveVal
-      }
-    };
   } else if (
-    interactiveValue.interactive.type === 'userInput' &&
+    finalInteractive.type === 'userInput' &&
     typeof parsedUserInteractiveVal === 'object'
   ) {
-    interactiveValue.interactive = {
+    finalInteractive.params.inputForm = finalInteractive.params.inputForm.map((item) => {
-      ...interactiveValue.interactive,
+      const itemValue = parsedUserInteractiveVal[item.label];
-      params: {
+      return itemValue !== undefined
-        ...interactiveValue.interactive.params,
+        ? {
-        inputForm: interactiveValue.interactive.params.inputForm.map((item) => {
+            ...item,
-          const itemValue = parsedUserInteractiveVal[item.label];
+            value: itemValue
-          return itemValue !== undefined
+          }
-            ? {
+        : item;
-                ...item,
+    });
-                value: itemValue
+    finalInteractive.params.submitted = true;
-              }
-            : item;
-        }),
-        submitted: true
-      }
-    };
   }
 
   if (aiResponse.customFeedbacks) {

packages/service/core/dataset/apiDataset/proApi.ts

@@ -0,0 +1,25 @@
+import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
+import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
+
+export enum ProApiDatasetOperationTypeEnum {
+  LIST = 'list',
+  READ = 'read',
+  CONTENT = 'content'
+}
+
+export type ProApiDatasetCommonParams = {
+  feishuServer?: FeishuServer;
+  yuqueServer?: YuqueServer;
+};
+
+export type GetProApiDatasetFileListParams = ProApiDatasetCommonParams & {
+  parentId?: ParentIdType;
+};
+
+export type GetProApiDatasetFileContentParams = ProApiDatasetCommonParams & {
+  apiFileId: string;
+};
+
+export type GetProApiDatasetFilePreviewUrlParams = ProApiDatasetCommonParams & {
+  apiFileId: string;
+};

packages/service/core/dataset/collection/controller.ts

@@ -1,6 +1,7 @@
 import {
   DatasetCollectionTypeEnum,
-  DatasetCollectionDataProcessModeEnum
+  DatasetCollectionDataProcessModeEnum,
+  DatasetTypeEnum
 } from '@fastgpt/global/core/dataset/constants';
 import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
 import { MongoDatasetCollection } from './schema';
@@ -104,7 +105,8 @@ export const createCollectionAndInsertData = async ({
       hashRawText: hashStr(rawText),
       rawTextLength: rawText.length,
       nextSyncTime: (() => {
-        if (!dataset.autoSync) return undefined;
+        // ignore auto collections sync for website datasets
+        if (!dataset.autoSync && dataset.type === DatasetTypeEnum.websiteDataset) return undefined;
         if (
           [DatasetCollectionTypeEnum.link, DatasetCollectionTypeEnum.apiFile].includes(
             createCollectionParams.type

packages/service/core/dataset/collection/schema.ts

@@ -1,13 +1,8 @@
 import { connectionMongo, getMongoModel } from '../../../common/mongo';
-const { Schema, model, models } = connectionMongo;
+const { Schema } = connectionMongo;
 import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
-import {
+import { DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constants';
-  DatasetCollectionTypeMap,
+import { ChunkSettings, DatasetCollectionName } from '../schema';
-  DatasetCollectionDataProcessModeEnum,
-  ChunkSettingModeEnum,
-  DataChunkSplitModeEnum
-} from '@fastgpt/global/core/dataset/constants';
-import { DatasetCollectionName } from '../schema';
 import {
   TeamCollectionName,
   TeamMemberCollectionName
@@ -90,25 +85,7 @@ const DatasetCollectionSchema = new Schema({
   customPdfParse: Boolean,
 
   // Chunk settings
-  imageIndex: Boolean,
+  ...ChunkSettings
-  autoIndexes: Boolean,
-  trainingType: {
-    type: String,
-    enum: Object.values(DatasetCollectionDataProcessModeEnum)
-  },
-  chunkSettingMode: {
-    type: String,
-    enum: Object.values(ChunkSettingModeEnum)
-  },
-  chunkSplitMode: {
-    type: String,
-    enum: Object.values(DataChunkSplitModeEnum)
-  },
-  chunkSize: Number,
-  chunkSplitter: String,
-
-  indexSize: Number,
-  qaPrompt: String
 });
 
 DatasetCollectionSchema.virtual('dataset', {

packages/service/core/dataset/controller.ts

@@ -9,6 +9,8 @@ import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
 import { MongoDatasetDataText } from './data/dataTextSchema';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { retryFn } from '@fastgpt/global/common/system/utils';
+import { removeWebsiteSyncJobScheduler } from './websiteSync';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
 
 /* ============= dataset ========== */
 /* find all datasetId by top datasetId */

packages/service/core/dataset/read.ts

@@ -9,7 +9,6 @@ import { readRawContentByFileBuffer } from '../../common/file/read/utils';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { APIFileServer, FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
 import { useApiDatasetRequest } from './apiDataset/api';
-import { POST } from '../../common/api/plusRequest';
 
 export const readFileRawTextByUrl = async ({
   teamId,
@@ -168,11 +167,7 @@ export const readApiServerFileContent = async ({
   }
 
   if (feishuServer || yuqueServer) {
-    return POST<{
+    return global.getProApiDatasetFileContent({
-      title?: string;
-      rawText: string;
-    }>(`/core/dataset/systemApiDataset`, {
-      type: 'content',
       feishuServer,
       yuqueServer,
       apiFileId

packages/service/core/dataset/schema.ts

@@ -1,7 +1,8 @@
 import { getMongoModel, Schema } from '../../common/mongo';
 import {
-  DatasetStatusEnum,
+  ChunkSettingModeEnum,
-  DatasetStatusMap,
+  DataChunkSplitModeEnum,
+  DatasetCollectionDataProcessModeEnum,
   DatasetTypeEnum,
   DatasetTypeMap
 } from '@fastgpt/global/core/dataset/constants';
@@ -13,6 +14,28 @@ import type { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
 
 export const DatasetCollectionName = 'datasets';
 
+export const ChunkSettings = {
+  imageIndex: Boolean,
+  autoIndexes: Boolean,
+  trainingType: {
+    type: String,
+    enum: Object.values(DatasetCollectionDataProcessModeEnum)
+  },
+  chunkSettingMode: {
+    type: String,
+    enum: Object.values(ChunkSettingModeEnum)
+  },
+  chunkSplitMode: {
+    type: String,
+    enum: Object.values(DataChunkSplitModeEnum)
+  },
+  chunkSize: Number,
+  chunkSplitter: String,
+
+  indexSize: Number,
+  qaPrompt: String
+};
+
 const DatasetSchema = new Schema({
   parentId: {
     type: Schema.Types.ObjectId,
@@ -40,11 +63,6 @@ const DatasetSchema = new Schema({
     required: true,
     default: DatasetTypeEnum.dataset
   },
-  status: {
-    type: String,
-    enum: Object.keys(DatasetStatusMap),
-    default: DatasetStatusEnum.active
-  },
   avatar: {
     type: String,
     default: '/icon/logo.svg'
@@ -84,6 +102,9 @@ const DatasetSchema = new Schema({
       }
     }
   },
+  chunkSettings: {
+    type: ChunkSettings
+  },
   inheritPermission: {
     type: Boolean,
     default: true
@@ -98,9 +119,8 @@ const DatasetSchema = new Schema({
     type: Object
   },
 
-  autoSync: Boolean,
-
   // abandoned
+  autoSync: Boolean,
   externalReadUrl: {
     type: String
   },

packages/service/core/dataset/search/controller.ts

@@ -24,7 +24,6 @@ import { MongoDatasetCollectionTags } from '../tag/schema';
 import { readFromSecondary } from '../../../common/mongo/utils';
 import { MongoDatasetDataText } from '../data/dataTextSchema';
 import { ChatItemType } from '@fastgpt/global/core/chat/type';
-import { POST } from '../../../common/api/plusRequest';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { datasetSearchQueryExtension } from './utils';
 import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
@@ -850,5 +849,4 @@ export type DeepRagSearchProps = SearchDatasetDataProps & {
   [NodeInputKeyEnum.datasetDeepSearchMaxTimes]?: number;
   [NodeInputKeyEnum.datasetDeepSearchBg]?: string;
 };
-export const deepRagSearch = (data: DeepRagSearchProps) =>
+export const deepRagSearch = (data: DeepRagSearchProps) => global.deepRagHandler(data);
-  POST<SearchDatasetDataResponse>('/core/dataset/deepRag', data);

packages/service/core/dataset/training/schema.ts

@@ -98,7 +98,9 @@ const TrainingDataSchema = new Schema({
       }
     ],
     default: []
-  }
+  },
+
+  errorMsg: String
 });
 
 TrainingDataSchema.virtual('dataset', {

packages/service/core/dataset/websiteSync/index.ts

@@ -0,0 +1,101 @@
+import { Processor } from 'bullmq';
+import { getQueue, getWorker, QueueNames } from '../../../common/bullmq';
+import { DatasetStatusEnum } from '@fastgpt/global/core/dataset/constants';
+
+export type WebsiteSyncJobData = {
+  datasetId: string;
+};
+
+export const websiteSyncQueue = getQueue<WebsiteSyncJobData>(QueueNames.websiteSync, {
+  defaultJobOptions: {
+    attempts: 3, // retry 3 times
+    backoff: {
+      type: 'exponential',
+      delay: 1000 // delay 1 second between retries
+    }
+  }
+});
+export const getWebsiteSyncWorker = (processor: Processor<WebsiteSyncJobData>) => {
+  return getWorker<WebsiteSyncJobData>(QueueNames.websiteSync, processor, {
+    removeOnFail: {
+      age: 15 * 24 * 60 * 60, // Keep up to 15 days
+      count: 1000 // Keep up to 1000 jobs
+    },
+    concurrency: 1 // Set worker to process only 1 job at a time
+  });
+};
+
+export const addWebsiteSyncJob = (data: WebsiteSyncJobData) => {
+  const datasetId = String(data.datasetId);
+  // deduplication: make sure only 1 job
+  return websiteSyncQueue.add(datasetId, data, { deduplication: { id: datasetId } });
+};
+
+export const getWebsiteSyncDatasetStatus = async (datasetId: string) => {
+  const jobId = await websiteSyncQueue.getDeduplicationJobId(datasetId);
+  if (!jobId) {
+    return {
+      status: DatasetStatusEnum.active,
+      errorMsg: undefined
+    };
+  }
+  const job = await websiteSyncQueue.getJob(jobId);
+  if (!job) {
+    return {
+      status: DatasetStatusEnum.active,
+      errorMsg: undefined
+    };
+  }
+
+  const jobState = await job.getState();
+
+  if (jobState === 'failed' || jobState === 'unknown') {
+    return {
+      status: DatasetStatusEnum.error,
+      errorMsg: job.failedReason
+    };
+  }
+  if (['waiting-children', 'waiting'].includes(jobState)) {
+    return {
+      status: DatasetStatusEnum.waiting,
+      errorMsg: undefined
+    };
+  }
+  if (jobState === 'active') {
+    return {
+      status: DatasetStatusEnum.syncing,
+      errorMsg: undefined
+    };
+  }
+
+  return {
+    status: DatasetStatusEnum.active,
+    errorMsg: undefined
+  };
+};
+
+// Scheduler setting
+const repeatDuration = 24 * 60 * 60 * 1000; // every day
+export const upsertWebsiteSyncJobScheduler = (data: WebsiteSyncJobData, startDate?: number) => {
+  const datasetId = String(data.datasetId);
+
+  return websiteSyncQueue.upsertJobScheduler(
+    datasetId,
+    {
+      every: repeatDuration,
+      startDate: startDate || new Date().getTime() + repeatDuration // First run tomorrow
+    },
+    {
+      name: datasetId,
+      data
+    }
+  );
+};
+
+export const getWebsiteSyncJobScheduler = (datasetId: string) => {
+  return websiteSyncQueue.getJobScheduler(String(datasetId));
+};
+
+export const removeWebsiteSyncJobScheduler = (datasetId: string) => {
+  return websiteSyncQueue.removeJobScheduler(String(datasetId));
+};

packages/service/core/workflow/dispatch/agent/runTool/index.ts

@@ -29,9 +29,9 @@ import { InteractiveNodeResponseType } from '@fastgpt/global/core/workflow/templ
 import { getFileContentFromLinks, getHistoryFileLinks } from '../../tools/readFiles';
 import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
-import { postTextCensor } from '../../../../../common/api/requestPlusApi';
 import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
 import { getDocumentQuotePrompt } from '@fastgpt/global/core/ai/prompt/AIChat';
+import { postTextCensor } from '../../../../chat/postTextCensor';
 
 type Response = DispatchNodeResultType<{
   [NodeOutputKeyEnum.answerText]: string;
@@ -176,7 +176,8 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
     toolNodeOutputTokens,
     completeMessages = [], // The actual message sent to AI(just save text)
     assistantResponses = [], // FastGPT system store assistant.value response
-    runTimes
+    runTimes,
+    finish_reason
   } = await (async () => {
     const adaptMessages = chats2GPTMessages({
       messages,
@@ -276,7 +277,8 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
         useVision
       ),
       toolDetail: childToolResponse,
-      mergeSignId: nodeId
+      mergeSignId: nodeId,
+      finishReason: finish_reason
     },
     [DispatchNodeResponseKeyEnum.nodeDispatchUsages]: [
       // 工具调用本身的积分消耗

packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts

@@ -1,6 +1,10 @@
 import { createChatCompletion } from '../../../../ai/config';
 import { filterGPTMessageByMaxContext, loadRequestMessages } from '../../../../chat/utils';
-import { StreamChatType, ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type';
+import {
+  StreamChatType,
+  ChatCompletionMessageParam,
+  CompletionFinishReason
+} from '@fastgpt/global/core/ai/type';
 import { NextApiResponse } from 'next';
 import { responseWriteController } from '../../../../../common/response';
 import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
@@ -252,9 +256,9 @@ export const runToolWithPromptCall = async (
     }
   });
 
-  const { answer, reasoning } = await (async () => {
+  const { answer, reasoning, finish_reason } = await (async () => {
     if (res && isStreamResponse) {
-      const { answer, reasoning } = await streamResponse({
+      const { answer, reasoning, finish_reason } = await streamResponse({
         res,
         toolNodes,
         stream: aiResponse,
@@ -262,8 +266,9 @@ export const runToolWithPromptCall = async (
         aiChatReasoning
       });
 
-      return { answer, reasoning };
+      return { answer, reasoning, finish_reason };
     } else {
+      const finish_reason = aiResponse.choices?.[0]?.finish_reason as CompletionFinishReason;
       const content = aiResponse.choices?.[0]?.message?.content || '';
       const reasoningContent: string = aiResponse.choices?.[0]?.message?.reasoning_content || '';
 
@@ -271,14 +276,16 @@ export const runToolWithPromptCall = async (
       if (reasoningContent || !aiChatReasoning) {
         return {
           answer: content,
-          reasoning: reasoningContent
+          reasoning: reasoningContent,
+          finish_reason
         };
       }
 
       const [think, answer] = parseReasoningContent(content);
       return {
         answer,
-        reasoning: think
+        reasoning: think,
+        finish_reason
       };
     }
   })();
@@ -525,7 +532,8 @@ ANSWER: `;
       toolNodeInputTokens,
       toolNodeOutputTokens,
       assistantResponses: toolNodeAssistants,
-      runTimes
+      runTimes,
+      finish_reason
     }
   );
 };
@@ -550,15 +558,18 @@ async function streamResponse({
   let startResponseWrite = false;
   let answer = '';
   let reasoning = '';
+  let finish_reason: CompletionFinishReason = null;
   const { parsePart, getStartTagBuffer } = parseReasoningStreamContent();
 
   for await (const part of stream) {
     if (res.closed) {
       stream.controller?.abort();
+      finish_reason = 'close';
       break;
     }
 
-    const [reasoningContent, content] = parsePart(part, aiChatReasoning);
+    const { reasoningContent, content, finishReason } = parsePart(part, aiChatReasoning);
+    finish_reason = finish_reason || finishReason;
     answer += content;
     reasoning += reasoningContent;
 
@@ -618,7 +629,7 @@ async function streamResponse({
     }
   }
 
-  return { answer, reasoning };
+  return { answer, reasoning, finish_reason };
 }
 
 const parseAnswer = (

packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts

@@ -7,7 +7,8 @@ import {
   ChatCompletionToolMessageParam,
   ChatCompletionMessageParam,
   ChatCompletionTool,
-  ChatCompletionAssistantMessageParam
+  ChatCompletionAssistantMessageParam,
+  CompletionFinishReason
 } from '@fastgpt/global/core/ai/type';
 import { NextApiResponse } from 'next';
 import { responseWriteController } from '../../../../../common/response';
@@ -300,7 +301,7 @@ export const runToolWithToolChoice = async (
     }
   });
 
-  const { answer, toolCalls } = await (async () => {
+  const { answer, toolCalls, finish_reason } = await (async () => {
     if (res && isStreamResponse) {
       return streamResponse({
         res,
@@ -310,6 +311,7 @@ export const runToolWithToolChoice = async (
       });
     } else {
       const result = aiResponse as ChatCompletion;
+      const finish_reason = result.choices?.[0]?.finish_reason as CompletionFinishReason;
       const calls = result.choices?.[0]?.message?.tool_calls || [];
       const answer = result.choices?.[0]?.message?.content || '';
 
@@ -350,7 +352,8 @@ export const runToolWithToolChoice = async (
 
       return {
         answer,
-        toolCalls: toolCalls
+        toolCalls: toolCalls,
+        finish_reason
       };
     }
   })();
@@ -549,8 +552,9 @@ export const runToolWithToolChoice = async (
         toolNodeOutputTokens,
         completeMessages,
         assistantResponses: toolNodeAssistants,
+        toolWorkflowInteractiveResponse,
         runTimes,
-        toolWorkflowInteractiveResponse
+        finish_reason
       };
     }
 
@@ -565,7 +569,8 @@ export const runToolWithToolChoice = async (
         toolNodeInputTokens,
         toolNodeOutputTokens,
         assistantResponses: toolNodeAssistants,
-        runTimes
+        runTimes,
+        finish_reason
       }
     );
   } else {
@@ -588,7 +593,8 @@ export const runToolWithToolChoice = async (
 
       completeMessages,
       assistantResponses: [...assistantResponses, ...toolNodeAssistant.value],
-      runTimes: (response?.runTimes || 0) + 1
+      runTimes: (response?.runTimes || 0) + 1,
+      finish_reason
     };
   }
 };
@@ -612,14 +618,18 @@ async function streamResponse({
   let textAnswer = '';
   let callingTool: { name: string; arguments: string } | null = null;
   let toolCalls: ChatCompletionMessageToolCall[] = [];
+  let finishReason: CompletionFinishReason = null;
 
   for await (const part of stream) {
     if (res.closed) {
       stream.controller?.abort();
+      finishReason = 'close';
       break;
     }
 
     const responseChoice = part.choices?.[0]?.delta;
+    const finish_reason = part.choices?.[0]?.finish_reason as CompletionFinishReason;
+    finishReason = finishReason || finish_reason;
 
     if (responseChoice?.content) {
       const content = responseChoice.content || '';
@@ -705,5 +715,5 @@ async function streamResponse({
     }
   }
 
-  return { answer: textAnswer, toolCalls };
+  return { answer: textAnswer, toolCalls, finish_reason: finishReason };
 }

packages/service/core/workflow/dispatch/agent/runTool/type.d.ts

@@ -1,4 +1,4 @@
-import { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type';
+import { ChatCompletionMessageParam, CompletionFinishReason } from '@fastgpt/global/core/ai/type';
 import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import type {
   ModuleDispatchProps,
@@ -43,6 +43,7 @@ export type RunToolResponse = {
   assistantResponses?: AIChatItemValueItemType[];
   toolWorkflowInteractiveResponse?: WorkflowInteractiveResponseType;
   [DispatchNodeResponseKeyEnum.runTimes]: number;
+  finish_reason?: CompletionFinishReason;
 };
 export type ToolNodeItemType = RuntimeNodeItemType & {
   toolParams: RuntimeNodeItemType['inputs'];

packages/service/core/workflow/dispatch/chat/oneapi.ts

@@ -6,10 +6,13 @@ import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/cons
 import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils';
 import { parseReasoningContent, parseReasoningStreamContent } from '../../../ai/utils';
 import { createChatCompletion } from '../../../ai/config';
-import type { ChatCompletionMessageParam, StreamChatType } from '@fastgpt/global/core/ai/type.d';
+import type {
+  ChatCompletionMessageParam,
+  CompletionFinishReason,
+  StreamChatType
+} from '@fastgpt/global/core/ai/type.d';
 import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
 import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
-import { postTextCensor } from '../../../../common/api/requestPlusApi';
 import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
 import type {
   ChatDispatchProps,
@@ -47,6 +50,7 @@ import { getFileContentFromLinks, getHistoryFileLinks } from '../tools/readFiles
 import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
 import { i18nT } from '../../../../../web/i18n/utils';
 import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
+import { postTextCensor } from '../../../chat/postTextCensor';
 
 export type ChatProps = ModuleDispatchProps<
   AIChatNodeProps & {
@@ -101,7 +105,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
 
   const modelConstantsData = getLLMModel(model);
   if (!modelConstantsData) {
-    return Promise.reject('The chat model is undefined, you need to select a chat model.');
+    return Promise.reject(`Mode ${model} is undefined, you need to select a chat model.`);
   }
 
   aiChatVision = modelConstantsData.vision && aiChatVision;
@@ -195,16 +199,17 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
     }
   });
 
-  const { answerText, reasoningText } = await (async () => {
+  const { answerText, reasoningText, finish_reason } = await (async () => {
     if (isStreamResponse) {
       if (!res) {
         return {
           answerText: '',
-          reasoningText: ''
+          reasoningText: '',
+          finish_reason: 'close' as const
         };
       }
       // sse response
-      const { answer, reasoning } = await streamResponse({
+      const { answer, reasoning, finish_reason } = await streamResponse({
         res,
         stream: response,
         aiChatReasoning,
@@ -215,9 +220,12 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
 
       return {
         answerText: answer,
-        reasoningText: reasoning
+        reasoningText: reasoning,
+        finish_reason
       };
     } else {
+      const finish_reason = response.choices?.[0]?.finish_reason as CompletionFinishReason;
+
       const { content, reasoningContent } = (() => {
         const content = response.choices?.[0]?.message?.content || '';
         // @ts-ignore
@@ -260,7 +268,8 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
 
       return {
         answerText: content,
-        reasoningText: reasoningContent
+        reasoningText: reasoningContent,
+        finish_reason
       };
     }
   })();
@@ -303,7 +312,8 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
       maxToken: max_tokens,
       reasoningText,
       historyPreview: getHistoryPreview(chatCompleteMessages, 10000, aiChatVision),
-      contextTotalLen: completeMessages.length
+      contextTotalLen: completeMessages.length,
+      finishReason: finish_reason
     },
     [DispatchNodeResponseKeyEnum.nodeDispatchUsages]: [
       {
@@ -528,15 +538,18 @@ async function streamResponse({
   });
   let answer = '';
   let reasoning = '';
+  let finish_reason: CompletionFinishReason = null;
   const { parsePart, getStartTagBuffer } = parseReasoningStreamContent();
 
   for await (const part of stream) {
     if (res.closed) {
       stream.controller?.abort();
+      finish_reason = 'close';
       break;
     }
 
-    const [reasoningContent, content] = parsePart(part, parseThinkTag);
+    const { reasoningContent, content, finishReason } = parsePart(part, parseThinkTag);
+    finish_reason = finish_reason || finishReason;
     answer += content;
     reasoning += reasoningContent;
 
@@ -575,5 +588,5 @@ async function streamResponse({
     }
   }
 
-  return { answer, reasoning };
+  return { answer, reasoning, finish_reason };
 }