update doc (#4460)

* update preview action

* update doc

* remove

* update

* schema

.github/sync_imgs.yml

@@ -1,4 +1,4 @@
 yangchuansheng/fastgpt-imgs:
   - source: docSite/assets/imgs/
     dest: imgs/
-    deleteOrphaned: true
+    deleteOrphaned: true

.github/workflows/docs-deploy-vercel.yml

@@ -1,4 +1,4 @@
-name: Deploy doc image to cf
+name: Deploy doc image to vercel
 
 on:
   workflow_dispatch:
@@ -63,6 +63,18 @@ jobs:
       - name: Build
         run: cd docSite && hugo mod get -u github.com/colinwilson/lotusdocs@6d0568e && hugo -v --minify
 
+      # Step 5 - Push our generated site to Vercel
+      - name: Deploy to Vercel
+        uses: amondnet/vercel-action@v25
+        id: vercel-action
+        with:
+          vercel-token: ${{ secrets.VERCEL_TOKEN }} # Required
+          vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} #Required
+          vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }} #Required
+          github-comment: false
+          vercel-args: '--prod --local-config ../vercel.json' # Optional
+          working-directory: docSite/public
+
       - name: Deploy to GitHub Pages
         uses: peaceiris/actions-gh-pages@v4
         if: github.ref == 'refs/heads/main'

.github/workflows/docs-preview.yml

@@ -1,7 +1,7 @@
 name: Preview FastGPT docs
 
 on:
-  pull_request_target:
+  pull_request:
     paths:
       - 'docSite/**'
   workflow_dispatch:
@@ -50,6 +50,10 @@ jobs:
               - 'docSite/content/docs/**'
           base: main
 
+      - name: Add cdn for images
+        run: |
+          sed -i "s#\](/imgs/#\](https://cdn.jsdelivr.net/gh/yangchuansheng/fastgpt-imgs@main/imgs/#g" $(grep -rl "\](/imgs/" docSite/content/zh-cn/docs)
+
       # Step 3 - Install Hugo (specific version)
       - name: Install Hugo
         uses: peaceiris/actions-hugo@v2
@@ -61,6 +65,9 @@ jobs:
       - name: Build
         run: cd docSite && hugo mod get -u github.com/colinwilson/lotusdocs@6d0568e && hugo -v --minify
 
+      - name: Test
+        run: ls ./docSite/public
+
       # Step 5 - Push our generated site to Cloudflare
       - name: Deploy to Cloudflare Pages
         id: deploy

docSite/content/zh-cn/docs/development/openapi/chat.md

@@ -34,94 +34,6 @@ weight: 852
 
 ### 请求
 
-{{< tabs tabTotal="3" >}}
-{{< tab tabName="基础请求示例" >}}
-{{< markdownify >}}
-
-```bash
-curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
---header 'Authorization: Bearer fastgpt-xxxxxx' \
---header 'Content-Type: application/json' \
---data-raw '{
-    "chatId": "my_chatId",
-    "stream": false,
-    "detail": false,
-    "responseChatItemId": "my_responseChatItemId",
-    "variables": {
-        "uid": "asdfadsfasfd2323",
-        "name": "张三"
-    },
-    "messages": [
-        {
-            "role": "user",
-            "content": "导演是谁"
-        }
-    ]
-}'
-```
-
-{{< /markdownify >}}
-{{< /tab >}}
-
-{{< tab tabName="图片/文件请求示例" >}}
-{{< markdownify >}}
-
-* 仅`messages`有部分区别，其他参数一致。
-* 目前不支持上传文件，需上传到自己的对象存储中，获取对应的文件链接。
-
-```bash
-curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
---header 'Authorization: Bearer fastgpt-xxxxxx' \
---header 'Content-Type: application/json' \
---data-raw '{
-    "chatId": "abcd",
-    "stream": false,
-    "messages": [
-        {
-            "role": "user",
-            "content": [
-                {
-                    "type": "text",
-                    "text": "导演是谁"
-                },
-                {
-                    "type": "image_url",
-                    "image_url": {
-                        "url": "图片链接"
-                    }
-                },
-                {
-                    "type": "file_url",
-                    "name": "文件名",
-                    "url": "文档链接，支持 txt md html word pdf ppt csv excel"
-                }
-            ]
-        }
-    ]
-}'
-```
-
-{{< /markdownify >}}
-{{< /tab >}}
-
-{{< tab tabName="参数说明" >}}
-{{< markdownify >}}
-
-{{% alert context="info" %}}
-- headers.Authorization: Bearer {{apikey}}
-- chatId: string | undefined 。
-  - 为 `undefined` 时（不传入），不使用 FastGpt 提供的上下文功能，完全通过传入的 messages 构建上下文。
-  - 为`非空字符串`时，意味着使用 chatId 进行对话，自动从 FastGpt 数据库取历史记录，并使用 messages 数组最后一个内容作为用户问题，其余 message 会被忽略。请自行确保 chatId 唯一，长度小于250，通常可以是自己系统的对话框ID。
-- messages: 结构与 [GPT接口](https://platform.openai.com/docs/api-reference/chat/object) chat模式一致。
-- responseChatItemId: string | undefined 。如果传入，则会将该值作为本次对话的响应消息的 ID，FastGPT 会自动将该 ID 存入数据库。请确保，在当前`chatId`下，`responseChatItemId`是唯一的。
-- detail: 是否返回中间值（模块状态，响应的完整结果等），`stream模式`下会通过`event`进行区分，`非stream模式`结果保存在`responseData`中。
-- variables: 模块变量，一个对象，会替换模块中，输入框内容里的`{{key}}`
-{{% /alert %}}
-
-{{< /markdownify >}}
-{{< /tab >}}
-{{< /tabs >}}
-
 <!-- #### v2
 
 v1,v2 接口请求参数一致，仅请求地址不一样。
@@ -216,7 +128,93 @@ curl --location --request POST 'http://localhost:3000/api/v2/chat/completions' \
 
 #### v1
 
+{{< tabs tabTotal="3" >}}
+{{< tab tabName="基础请求示例" >}}
+{{< markdownify >}}
 
+```bash
+curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
+--header 'Authorization: Bearer fastgpt-xxxxxx' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+    "chatId": "my_chatId",
+    "stream": false,
+    "detail": false,
+    "responseChatItemId": "my_responseChatItemId",
+    "variables": {
+        "uid": "asdfadsfasfd2323",
+        "name": "张三"
+    },
+    "messages": [
+        {
+            "role": "user",
+            "content": "导演是谁"
+        }
+    ]
+}'
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="图片/文件请求示例" >}}
+{{< markdownify >}}
+
+* 仅`messages`有部分区别，其他参数一致。
+* 目前不支持上传文件，需上传到自己的对象存储中，获取对应的文件链接。
+
+```bash
+curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
+--header 'Authorization: Bearer fastgpt-xxxxxx' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+    "chatId": "abcd",
+    "stream": false,
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "导演是谁"
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "图片链接"
+                    }
+                },
+                {
+                    "type": "file_url",
+                    "name": "文件名",
+                    "url": "文档链接，支持 txt md html word pdf ppt csv excel"
+                }
+            ]
+        }
+    ]
+}'
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="参数说明" >}}
+{{< markdownify >}}
+
+{{% alert context="info" %}}
+- headers.Authorization: Bearer {{apikey}}
+- chatId: string | undefined 。
+  - 为 `undefined` 时（不传入），不使用 FastGpt 提供的上下文功能，完全通过传入的 messages 构建上下文。
+  - 为`非空字符串`时，意味着使用 chatId 进行对话，自动从 FastGpt 数据库取历史记录，并使用 messages 数组最后一个内容作为用户问题，其余 message 会被忽略。请自行确保 chatId 唯一，长度小于250，通常可以是自己系统的对话框ID。
+- messages: 结构与 [GPT接口](https://platform.openai.com/docs/api-reference/chat/object) chat模式一致。
+- responseChatItemId: string | undefined 。如果传入，则会将该值作为本次对话的响应消息的 ID，FastGPT 会自动将该 ID 存入数据库。请确保，在当前`chatId`下，`responseChatItemId`是唯一的。
+- detail: 是否返回中间值（模块状态，响应的完整结果等），`stream模式`下会通过`event`进行区分，`非stream模式`结果保存在`responseData`中。
+- variables: 模块变量，一个对象，会替换模块中，输入框内容里的`{{key}}`
+{{% /alert %}}
+
+{{< /markdownify >}}
+{{< /tab >}}
+{{< /tabs >}}
 
 ### 响应
 
@@ -747,6 +745,8 @@ curl --location --request POST 'https://api.fastgpt.in/api/v1/chat/completions'
 
 ### 请求示例
 
+#### v1
+
 ```bash
 curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
 --header 'Authorization: Bearer test-xxxxx' \
@@ -760,8 +760,25 @@ curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
 }'
 ```
 
+#### v2
+
+```bash
+curl --location --request POST 'http://localhost:3000/api/v2/chat/completions' \
+--header 'Authorization: Bearer test-xxxxx' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+    "stream": false,
+    "chatId": "test",
+    "variables": {
+        "query":"你好"
+    }
+}'
+```
+
 ### 响应示例
 
+#### v1
+
 {{< tabs tabTotal="3" >}}
 
 {{< tab tabName="detail=true,stream=false 响应" >}}
@@ -920,6 +937,151 @@ event取值：
 {{< /tab >}}
 {{< /tabs >}}
 
+
+#### v2
+
+{{< tabs tabTotal="3" >}}
+
+{{< tab tabName="detail=true,stream=false 响应" >}}
+{{< markdownify >}}
+
+* 插件的输出可以通过查找`responseData`中, `moduleType=pluginOutput`的元素，其`pluginOutput`是插件的输出。
+* 流输出，仍可以通过`choices`进行获取。
+
+```json
+{
+    "responseData": [
+        {
+            "id": "bsH1ZdbYkz9iJwYa",
+            "nodeId": "pluginInput",
+            "moduleName": "workflow:template.plugin_start",
+            "moduleType": "pluginInput",
+            "runningTime": 0
+        },
+        {
+            "id": "zDgfqSPhbYZFHVIn",
+            "nodeId": "h4Gr4lJtFVQ6qI4c",
+            "moduleName": "AI 对话",
+            "moduleType": "chatNode",
+            "runningTime": 1.44,
+            "totalPoints": 0,
+            "model": "GPT-4o-mini",
+            "tokens": 34,
+            "inputTokens": 8,
+            "outputTokens": 26,
+            "query": "你好",
+            "reasoningText": "",
+            "historyPreview": [
+                {
+                    "obj": "Human",
+                    "value": "你好"
+                },
+                {
+                    "obj": "AI",
+                    "value": "你好！有什么我可以帮助你的吗？"
+                }
+            ],
+            "contextTotalLen": 2
+        },
+        {
+            "id": "uLLwKKRZvufXzgF4",
+            "nodeId": "pluginOutput",
+            "moduleName": "common:core.module.template.self_output",
+            "moduleType": "pluginOutput",
+            "runningTime": 0,
+            "totalPoints": 0,
+            "pluginOutput": {
+                "result": "你好！有什么我可以帮助你的吗？"
+            }
+        }
+    ],
+    "newVariables": {
+
+    },
+    "id": "test",
+    "model": "",
+    "usage": {
+        "prompt_tokens": 1,
+        "completion_tokens": 1,
+        "total_tokens": 1
+    },
+    "choices": [
+        {
+            "message": {
+                "role": "assistant",
+                "content": "你好！有什么我可以帮助你的吗？"
+            },
+            "finish_reason": "stop",
+            "index": 0
+        }
+    ]
+}
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+
+{{< tab tabName="detail=true,stream=true 响应" >}}
+{{< markdownify >}}
+
+* 插件的输出可以通过获取`event=flowResponses`中的字符串，并将其反序列化后得到一个数组。同样的，查找 `moduleType=pluginOutput`的元素，其`pluginOutput`是插件的输出。
+* 流输出，仍和对话接口一样获取。
+
+```bash
+data: {"event":"flowNodeResponse","data":"{\"id\":\"q8ablUOqHGgqLIRM\",\"nodeId\":\"pluginInput\",\"moduleName\":\"workflow:template.plugin_start\",\"moduleType\":\"pluginInput\",\"runningTime\":0}"}
+
+data: {"event":"flowNodeStatus","data":"{\"status\":\"running\",\"name\":\"AI 对话\"}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"你好\"},\"index\":0,\"finish_reason\":null}]}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"！\"},\"index\":0,\"finish_reason\":null}]}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"有什么\"},\"index\":0,\"finish_reason\":null}]}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"我\"},\"index\":0,\"finish_reason\":null}]}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"可以\"},\"index\":0,\"finish_reason\":null}]}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"帮助\"},\"index\":0,\"finish_reason\":null}]}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"你\"},\"index\":0,\"finish_reason\":null}]}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"的吗\"},\"index\":0,\"finish_reason\":null}]}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":\"？\"},\"index\":0,\"finish_reason\":null}]}"}
+
+data: {"event":"flowNodeResponse","data":"{\"id\":\"rqlXLUap8QeiN7Kf\",\"nodeId\":\"h4Gr4lJtFVQ6qI4c\",\"moduleName\":\"AI 对话\",\"moduleType\":\"chatNode\",\"runningTime\":1.79,\"totalPoints\":0,\"model\":\"GPT-4o-mini\",\"tokens\":137,\"inputTokens\":111,\"outputTokens\":26,\"query\":\"你好\",\"reasoningText\":\"\",\"historyPreview\":[{\"obj\":\"Human\",\"value\":\"[{\\\"renderTypeList\\\":[\\\"reference\\\"],\\\"selectedTypeInde\\n\\n...[hide 174 chars]...\\n\\ncanSelectImg\\\":true,\\\"required\\\":false,\\\"value\\\":\\\"你好\\\"}]\"},{\"obj\":\"AI\",\"value\":\"你好！有什么我可以帮助你的吗？\"},{\"obj\":\"Human\",\"value\":\"你好\"},{\"obj\":\"AI\",\"value\":\"你好！有什么我可以帮助你的吗？\"}],\"contextTotalLen\":4}"}
+
+data: {"event":"flowNodeResponse","data":"{\"id\":\"lHCpHI0MrM00HQlX\",\"nodeId\":\"pluginOutput\",\"moduleName\":\"common:core.module.template.self_output\",\"moduleType\":\"pluginOutput\",\"runningTime\":0,\"totalPoints\":0,\"pluginOutput\":{\"result\":\"你好！有什么我可以帮助你的吗？\"}}"}
+
+data: {"event":"answer","data":"{\"id\":\"\",\"object\":\"\",\"created\":0,\"model\":\"\",\"choices\":[{\"delta\":{\"role\":\"assistant\",\"content\":null},\"index\":0,\"finish_reason\":\"stop\"}]}"}
+
+data: {"event":"answer","data":"[DONE]"}
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="输出获取" >}}
+{{< markdownify >}}
+
+event取值：
+
+- answer: 返回给客户端的文本（最终会算作回答）
+- fastAnswer: 指定回复返回给客户端的文本（最终会算作回答）
+- toolCall: 执行工具
+- toolParams: 工具参数
+- toolResponse: 工具返回
+- flowNodeStatus: 运行到的节点状态
+- flowNodeResponse: 单个节点详细响应
+- updateVariables: 更新变量
+- error: 报错
+
+{{< /markdownify >}}
+{{< /tab >}}
+{{< /tabs >}}
+
 # 对话 CRUD
 
 {{% alert icon="🤖 " context="success" %}}

docSite/content/zh-cn/docs/development/upgrading/494.md

@@ -22,10 +22,6 @@ weight: 796
 
 ### 2. 更新镜像 tag
 
-- 更新 FastGPT 镜像 tag: v4.9.4-alpha
-- 更新 FastGPT 商业版镜像 tag: v4.9.4-alpha
-- Sandbox 无需更新
-- AIProxy 无需更新
 
 ### 3. 执行升级脚本
 
@@ -51,7 +47,6 @@ curl --location --request POST 'https://{{host}}/api/admin/initv494' \
 4. 利用 redis 进行部分数据缓存。
 5. 站点同步支持配置训练参数。
 6. AI 对话/工具调用，增加返回模型 finish_reason 字段。
-7. 移动端语音输入交互调整
 
 ## ⚙️ 优化
 
@@ -61,6 +56,4 @@ curl --location --request POST 'https://{{host}}/api/admin/initv494' \
 
 ## 🐛 修复
 
-1. 搜索应用/知识库时，无法点击目录进入下一层。
+1. 搜索应用/知识库时，无法点击目录进入下一层。
-2. 重新训练时，参数未成功初始化。
-3. package/service 部分请求在多 app 中不一致。

docSite/content/zh-cn/docs/intro.md

@@ -7,7 +7,7 @@ toc: true
 weight: -10
 ---
 
-FastGPT 是一个AI Agent 构建平台，提供开箱即用的数据处理、模型调用等能力，同时可以通过 Flow 可视化进行工作流编排，从而实现复杂的应用场景！
+FastGPT 是一个AI Agent 构建平台，提供开箱即用的数据处理、模型调用等能力，同时可以通过 Flow 可视化进行工作流编排，从而实现复杂的应用场景！t
 
 {{% alert icon="🤖 " context="success" %}}
 FastGPT 在线使用：[https://tryfastgpt.ai](https://tryfastgpt.ai)

packages/global/core/dataset/constants.ts

@@ -51,8 +51,7 @@ export const DatasetTypeMap = {
 export enum DatasetStatusEnum {
   active = 'active',
   syncing = 'syncing',
-  waiting = 'waiting',
+  waiting = 'waiting'
-  error = 'error'
 }
 export const DatasetStatusMap = {
   [DatasetStatusEnum.active]: {
@@ -63,9 +62,6 @@ export const DatasetStatusMap = {
   },
   [DatasetStatusEnum.waiting]: {
     label: i18nT('common:core.dataset.status.waiting')
-  },
-  [DatasetStatusEnum.error]: {
-    label: i18nT('dataset:status_error')
   }
 };
 

packages/global/core/dataset/type.d.ts

@@ -209,7 +209,6 @@ export type DatasetListItemType = {
 
 export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel' | 'vlmModel'> & {
   status: `${DatasetStatusEnum}`;
-  errorMsg?: string;
   vectorModel: EmbeddingModelItemType;
   agentModel: LLMModelItemType;
   vlmModel?: LLMModelItemType;

packages/service/common/api/requestPlusApi.ts

@@ -1,6 +1,7 @@
+import { POST } from './plusRequest';
+
 export const postTextCensor = (data: { text: string }) =>
-  global
+  POST<{ code?: number; message: string }>('/common/censor/check', data)
-    .textCensorHandler(data)
     .then((res) => {
       if (res?.code === 5000) {
         return Promise.reject(res);

packages/service/common/api/type.d.ts

@@ -1,29 +0,0 @@
-import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
-import {
-  DeepRagSearchProps,
-  SearchDatasetDataResponse
-} from '../../core/dataset/search/controller';
-import { AuthOpenApiLimitProps } from '../../support/openapi/auth';
-import { CreateUsageProps, ConcatUsageProps } from '@fastgpt/global/support/wallet/usage/api';
-import {
-  GetProApiDatasetFileContentParams,
-  GetProApiDatasetFileListParams,
-  GetProApiDatasetFilePreviewUrlParams
-} from '../../core/dataset/apiDataset/proApi';
-
-declare global {
-  var textCensorHandler: (params: { text: string }) => Promise<{ code: number; message?: string }>;
-  var deepRagHandler: (data: DeepRagSearchProps) => Promise<SearchDatasetDataResponse>;
-  var authOpenApiHandler: (data: AuthOpenApiLimitProps) => Promise<any>;
-  var createUsageHandler: (data: CreateUsageProps) => Promise<void>;
-  var concatUsageHandler: (data: ConcatUsageProps) => Promise<void>;
-
-  // API dataset
-  var getProApiDatasetFileList: (data: GetProApiDatasetFileListParams) => Promise<APIFileItem[]>;
-  var getProApiDatasetFileContent: (
-    data: GetProApiDatasetFileContentParams
-  ) => Promise<ApiFileReadContentResponse>;
-  var getProApiDatasetFilePreviewUrl: (
-    data: GetProApiDatasetFilePreviewUrlParams
-  ) => Promise<string>;
-}

packages/service/common/bullmq/index.ts

@@ -69,11 +69,6 @@ export function getWorker<DataType, ReturnType = void>(
   newWorker.on('error', (error) => {
     addLog.error(`MQ Worker [${name}]: ${error.message}`, error);
   });
-  newWorker.on('failed', (jobId, error) => {
-    addLog.error(`MQ Worker [${name}]: ${error.message}`, error);
-  });
   workers.set(name, newWorker);
   return newWorker;
 }
-
-export * from 'bullmq';

packages/service/common/system/constants.ts

@@ -1,3 +1,4 @@
 export const FastGPTProUrl = process.env.PRO_URL ? `${process.env.PRO_URL}/api` : '';
+export const isFastGPTMainService = !!process.env.PRO_URL;
 // @ts-ignore
 export const isFastGPTProService = () => !!global.systemConfig;

packages/service/common/vectorStore/controller.ts

@@ -7,8 +7,6 @@ import { EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { MILVUS_ADDRESS, PG_ADDRESS, OCEANBASE_ADDRESS } from './constants';
 import { MilvusCtrl } from './milvus/class';
 import { setRedisCache, getRedisCache, delRedisCache, CacheKeyEnum } from '../redis/cache';
-import { throttle } from 'lodash';
-import { retryFn } from '@fastgpt/global/common/system/utils';
 
 const getVectorObj = () => {
   if (PG_ADDRESS) return new PgVectorCtrl();
@@ -17,12 +15,7 @@ const getVectorObj = () => {
 
   return new PgVectorCtrl();
 };
-
 const getChcheKey = (teamId: string) => `${CacheKeyEnum.team_vector_count}:${teamId}`;
-const onDelCache = throttle((teamId: string) => delRedisCache(getChcheKey(teamId)), 30000, {
-  leading: true,
-  trailing: true
-});
 
 const Vector = getVectorObj();
 
@@ -56,28 +49,26 @@ export const insertDatasetDataVector = async ({
   query: string;
   model: EmbeddingModelItemType;
 }) => {
-  return retryFn(async () => {
+  const { vectors, tokens } = await getVectorsByText({
-    const { vectors, tokens } = await getVectorsByText({
+    model,
-      model,
+    input: query,
-      input: query,
+    type: 'db'
-      type: 'db'
-    });
-    const { insertId } = await Vector.insert({
-      ...props,
-      vector: vectors[0]
-    });
-
-    onDelCache(props.teamId);
-
-    return {
-      tokens,
-      insertId
-    };
   });
+  const { insertId } = await Vector.insert({
+    ...props,
+    vector: vectors[0]
+  });
+
+  delRedisCache(getChcheKey(props.teamId));
+
+  return {
+    tokens,
+    insertId
+  };
 };
 
 export const deleteDatasetDataVector = async (props: DelDatasetVectorCtrlProps) => {
   const result = await Vector.delete(props);
-  onDelCache(props.teamId);
+  delRedisCache(getChcheKey(props.teamId));
   return result;
 };

packages/service/core/dataset/apiDataset/proApi.ts

@@ -1,25 +0,0 @@
-import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
-import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
-
-export enum ProApiDatasetOperationTypeEnum {
-  LIST = 'list',
-  READ = 'read',
-  CONTENT = 'content'
-}
-
-export type ProApiDatasetCommonParams = {
-  feishuServer?: FeishuServer;
-  yuqueServer?: YuqueServer;
-};
-
-export type GetProApiDatasetFileListParams = ProApiDatasetCommonParams & {
-  parentId?: ParentIdType;
-};
-
-export type GetProApiDatasetFileContentParams = ProApiDatasetCommonParams & {
-  apiFileId: string;
-};
-
-export type GetProApiDatasetFilePreviewUrlParams = ProApiDatasetCommonParams & {
-  apiFileId: string;
-};

packages/service/core/dataset/read.ts

@@ -9,6 +9,7 @@ import { readRawContentByFileBuffer } from '../../common/file/read/utils';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
 import { APIFileServer, FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
 import { useApiDatasetRequest } from './apiDataset/api';
+import { POST } from '../../common/api/plusRequest';
 
 export const readFileRawTextByUrl = async ({
   teamId,
@@ -167,7 +168,11 @@ export const readApiServerFileContent = async ({
   }
 
   if (feishuServer || yuqueServer) {
-    return global.getProApiDatasetFileContent({
+    return POST<{
+      title?: string;
+      rawText: string;
+    }>(`/core/dataset/systemApiDataset`, {
+      type: 'content',
       feishuServer,
       yuqueServer,
       apiFileId

packages/service/core/dataset/search/controller.ts

@@ -24,6 +24,7 @@ import { MongoDatasetCollectionTags } from '../tag/schema';
 import { readFromSecondary } from '../../../common/mongo/utils';
 import { MongoDatasetDataText } from '../data/dataTextSchema';
 import { ChatItemType } from '@fastgpt/global/core/chat/type';
+import { POST } from '../../../common/api/plusRequest';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { datasetSearchQueryExtension } from './utils';
 import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
@@ -849,4 +850,5 @@ export type DeepRagSearchProps = SearchDatasetDataProps & {
   [NodeInputKeyEnum.datasetDeepSearchMaxTimes]?: number;
   [NodeInputKeyEnum.datasetDeepSearchBg]?: string;
 };
-export const deepRagSearch = (data: DeepRagSearchProps) => global.deepRagHandler(data);
+export const deepRagSearch = (data: DeepRagSearchProps) =>
+  POST<SearchDatasetDataResponse>('/core/dataset/deepRag', data);

packages/service/core/dataset/websiteSync/index.ts

@@ -34,44 +34,23 @@ export const addWebsiteSyncJob = (data: WebsiteSyncJobData) => {
 export const getWebsiteSyncDatasetStatus = async (datasetId: string) => {
   const jobId = await websiteSyncQueue.getDeduplicationJobId(datasetId);
   if (!jobId) {
-    return {
+    return DatasetStatusEnum.active;
-      status: DatasetStatusEnum.active,
-      errorMsg: undefined
-    };
   }
   const job = await websiteSyncQueue.getJob(jobId);
   if (!job) {
-    return {
+    return DatasetStatusEnum.active;
-      status: DatasetStatusEnum.active,
-      errorMsg: undefined
-    };
   }
 
   const jobState = await job.getState();
 
-  if (jobState === 'failed' || jobState === 'unknown') {
-    return {
-      status: DatasetStatusEnum.error,
-      errorMsg: job.failedReason
-    };
-  }
   if (['waiting-children', 'waiting'].includes(jobState)) {
-    return {
+    return DatasetStatusEnum.waiting;
-      status: DatasetStatusEnum.waiting,
-      errorMsg: undefined
-    };
   }
   if (jobState === 'active') {
-    return {
+    return DatasetStatusEnum.syncing;
-      status: DatasetStatusEnum.syncing,
-      errorMsg: undefined
-    };
   }
 
-  return {
+  return DatasetStatusEnum.active;
-    status: DatasetStatusEnum.active,
-    errorMsg: undefined
-  };
 };
 
 // Scheduler setting

packages/service/core/workflow/dispatch/agent/runTool/index.ts

@@ -29,9 +29,9 @@ import { InteractiveNodeResponseType } from '@fastgpt/global/core/workflow/templ
 import { getFileContentFromLinks, getHistoryFileLinks } from '../../tools/readFiles';
 import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
+import { postTextCensor } from '../../../../../common/api/requestPlusApi';
 import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
 import { getDocumentQuotePrompt } from '@fastgpt/global/core/ai/prompt/AIChat';
-import { postTextCensor } from '../../../../chat/postTextCensor';
 
 type Response = DispatchNodeResultType<{
   [NodeOutputKeyEnum.answerText]: string;

packages/service/core/workflow/dispatch/chat/oneapi.ts

@@ -13,6 +13,7 @@ import type {
 } from '@fastgpt/global/core/ai/type.d';
 import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';
 import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
+import { postTextCensor } from '../../../../common/api/requestPlusApi';
 import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constants';
 import type {
   ChatDispatchProps,
@@ -50,7 +51,6 @@ import { getFileContentFromLinks, getHistoryFileLinks } from '../tools/readFiles
 import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
 import { i18nT } from '../../../../../web/i18n/utils';
 import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
-import { postTextCensor } from '../../../chat/postTextCensor';
 
 export type ChatProps = ModuleDispatchProps<
   AIChatNodeProps & {

packages/service/core/workflow/dispatch/loop/runLoop.ts

@@ -33,7 +33,7 @@ export const dispatchLoop = async (props: Props): Promise<Response> => {
     ? Number(process.env.WORKFLOW_MAX_LOOP_TIMES)
     : 50;
   if (loopInputArray.length > maxLength) {
-    return Promise.reject(`Input array length cannot be greater than ${maxLength}`);
+    return Promise.reject('Input array length cannot be greater than 50');
   }
 
   const outputValueArr = [];

packages/service/package.json

@@ -34,7 +34,7 @@
     "node-cron": "^3.0.3",
     "node-xlsx": "^0.24.0",
     "papaparse": "5.4.1",
-    "pdfjs-dist": "4.10.38",
+    "pdfjs-dist": "4.4.168",
     "pg": "^8.10.0",
     "request-ip": "^3.3.0",
     "tiktoken": "1.0.17",

packages/service/support/openapi/auth.ts

@@ -1,6 +1,7 @@
 import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
 import { updateApiKeyUsedTime } from './tools';
 import { MongoOpenApi } from './schema';
+import { POST } from '../../common/api/plusRequest';
 import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
 
 export type AuthOpenApiLimitProps = { openApi: OpenApiSchema };
@@ -16,10 +17,11 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
     }
 
     // auth limit
+    // @ts-ignore
     if (global.feConfigs?.isPlus) {
-      await global.authOpenApiHandler({
+      await POST('/support/openapi/authLimit', {
         openApi
-      });
+      } as AuthOpenApiLimitProps);
     }
 
     updateApiKeyUsedTime(openApi._id);

packages/service/support/wallet/usage/controller.ts

@@ -1,21 +1,67 @@
 import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
 import { MongoUsage } from './schema';
-import { ClientSession } from '../../../common/mongo';
+import { ClientSession, Types } from '../../../common/mongo';
 import { addLog } from '../../../common/system/log';
 import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
 import { ConcatUsageProps, CreateUsageProps } from '@fastgpt/global/support/wallet/usage/api';
 import { i18nT } from '../../../../web/i18n/utils';
+import { pushConcatBillTask, pushReduceTeamAiPointsTask } from './utils';
+
+import { POST } from '../../../common/api/plusRequest';
+import { isFastGPTMainService } from '../../../common/system/constants';
 
 export async function createUsage(data: CreateUsageProps) {
   try {
-    await global.createUsageHandler(data);
+    // In FastGPT server
+    if (isFastGPTMainService) {
+      await POST('/support/wallet/usage/createUsage', data);
+    } else if (global.reduceAiPointsQueue) {
+      // In FastGPT pro server
+      await MongoUsage.create(data);
+      pushReduceTeamAiPointsTask({ teamId: data.teamId, totalPoints: data.totalPoints });
+
+      if (data.totalPoints === 0) {
+        addLog.info('0 totalPoints', data);
+      }
+    }
   } catch (error) {
     addLog.error('createUsage error', error);
   }
 }
 export async function concatUsage(data: ConcatUsageProps) {
   try {
-    await global.concatUsageHandler(data);
+    // In FastGPT server
+    if (isFastGPTMainService) {
+      await POST('/support/wallet/usage/concatUsage', data);
+    } else if (global.reduceAiPointsQueue) {
+      const {
+        teamId,
+        billId,
+        totalPoints = 0,
+        listIndex,
+        inputTokens = 0,
+        outputTokens = 0
+      } = data;
+
+      // billId is required and valid
+      if (!billId || !Types.ObjectId.isValid(billId)) return;
+
+      // In FastGPT pro server
+      pushConcatBillTask([
+        {
+          billId,
+          listIndex,
+          inputTokens,
+          outputTokens,
+          totalPoints
+        }
+      ]);
+      pushReduceTeamAiPointsTask({ teamId, totalPoints });
+
+      if (data.totalPoints === 0) {
+        addLog.info('0 totalPoints', data);
+      }
+    }
   } catch (error) {
     addLog.error('concatUsage error', error);
   }

packages/service/support/wallet/usage/utils.ts

@@ -1,5 +1,6 @@
 import { findAIModel } from '../../../core/ai/model';
 import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
+import { ConcatBillQueueItemType } from './type';
 
 export const formatModelChars2Points = ({
   model,
@@ -34,3 +35,20 @@ export const formatModelChars2Points = ({
     totalPoints
   };
 };
+
+export const pushReduceTeamAiPointsTask = ({
+  teamId,
+  totalPoints
+}: {
+  teamId: string;
+  totalPoints: number;
+}) => {
+  global.reduceAiPointsQueue.push({
+    teamId: String(teamId),
+    totalPoints
+  });
+};
+
+export const pushConcatBillTask = (data: ConcatBillQueueItemType[]) => {
+  global.concatBillQueue.push(...data);
+};

packages/web/i18n/en/dataset.json

@@ -97,7 +97,7 @@
   "permission.des.read": "View knowledge base content",
   "permission.des.write": "Ability to add and change knowledge base content",
   "preview_chunk": "Preview chunks",
-  "preview_chunk_empty": "File content is empty",
+  "preview_chunk_empty": "Unable to read the contents of the file",
   "preview_chunk_intro": "A total of {{total}} blocks, up to 10",
   "preview_chunk_not_selected": "Click on the file on the left to preview",
   "process.Auto_Index": "Automatic index generation",
@@ -124,7 +124,6 @@
   "split_sign_question": "question mark",
   "split_sign_semicolon": "semicolon",
   "start_sync_website_tip": "Confirm to start synchronizing data? \nThe old data will be deleted and retrieved again, please confirm!",
-  "status_error": "Running exception",
   "sync_collection_failed": "Synchronization collection error, please check whether the source file can be accessed normally",
   "sync_schedule": "Timing synchronization",
   "sync_schedule_tip": "Only existing collections will be synchronized. \nIncludes linked collections and all collections in the API knowledge base. \nThe system will poll for updates every day, and the specific update time cannot be determined.",

packages/web/i18n/zh-CN/dataset.json

@@ -97,7 +97,7 @@
   "permission.des.read": "可查看知识库内容",
   "permission.des.write": "可增加和变更知识库内容",
   "preview_chunk": "分块预览",
-  "preview_chunk_empty": "文件内容为空",
+  "preview_chunk_empty": "无法读取该文件内容",
   "preview_chunk_intro": "共 {{total}} 个分块，最多展示 10 个",
   "preview_chunk_not_selected": "点击左侧文件后进行预览",
   "process.Auto_Index": "自动索引生成",
@@ -125,7 +125,6 @@
   "split_sign_question": "问号",
   "split_sign_semicolon": "分号",
   "start_sync_website_tip": "确认开始同步数据？将会删除旧数据后重新获取，请确认！",
-  "status_error": "运行异常",
   "sync_collection_failed": "同步集合错误，请检查是否能正常访问源文件",
   "sync_schedule": "定时同步",
   "sync_schedule_tip": "仅会同步已存在的集合。包括链接集合以及 API 知识库里所有集合。系统会每天进行轮询更新，无法确定具体的更新时间。",

packages/web/i18n/zh-Hant/dataset.json

@@ -97,7 +97,7 @@
   "permission.des.read": "可檢視資料集內容",
   "permission.des.write": "可新增和變更資料集內容",
   "preview_chunk": "分塊預覽",
-  "preview_chunk_empty": "文件內容為空",
+  "preview_chunk_empty": "無法讀取該文件內容",
   "preview_chunk_intro": "共 {{total}} 個分塊，最多展示 10 個",
   "preview_chunk_not_selected": "點擊左側文件後進行預覽",
   "process.Auto_Index": "自動索引生成",
@@ -124,7 +124,6 @@
   "split_sign_question": "問號",
   "split_sign_semicolon": "分號",
   "start_sync_website_tip": "確認開始同步資料？\n將會刪除舊資料後重新獲取，請確認！",
-  "status_error": "運行異常",
   "sync_collection_failed": "同步集合錯誤，請檢查是否能正常存取來源文件",
   "sync_schedule": "定時同步",
   "sync_schedule_tip": "只會同步已存在的集合。\n包括連結集合以及 API 知識庫裡所有集合。\n系統會每天進行輪詢更新，無法確定特定的更新時間。",

patches/mdast-util-gfm-autolink-literal@2.0.1.patch

@@ -1,79 +0,0 @@
-diff --git a/lib/index.js b/lib/index.js
-index c5ca771c24dd914e342f791716a822431ee32b3a..457d9f8c4625f7d9c7ea1e9ffc13616db1fc6fef 100644
---- a/lib/index.js
-+++ b/lib/index.js
-@@ -126,8 +126,37 @@ function exitLiteralAutolink(token) {
-   this.exit(token)
- }
- 
--/** @type {FromMarkdownTransform} */
-+// Regex support detector, for backward compatibility
-+// Ref: https://github.com/syntax-tree/mdast-util-gfm-autolink-literal/pull/14
-+const regexSupport = {
-+  lookbehind: (() => {
-+    try {
-+      // Using regex literal instead of RegExp constructor
-+      ;/(?<=x)/.test('x')
-+      return true
-+    } catch {
-+      return false
-+    }
-+  })()
-+}
-+
-+/**
-+ * Main transform function that uses the appropriate version based on regex support
-+ * @type {FromMarkdownTransform}
-+ */
- function transformGfmAutolinkLiterals(tree) {
-+  if (regexSupport.lookbehind) {
-+    modernAutolinkTransform(tree)
-+  } else {
-+    legacyAutolinkTransform(tree)
-+  }
-+}
-+
-+/**
-+ * Modern version of autolink transform using lookbehind
-+ * @type {FromMarkdownTransform}
-+ */
-+function modernAutolinkTransform(tree) {
-   findAndReplace(
-     tree,
-     [
-@@ -138,6 +167,35 @@ function transformGfmAutolinkLiterals(tree) {
-   )
- }
- 
-+
-+/**
-+ * Legacy version of autolink transform for older Node.js versions
-+ * @type {FromMarkdownTransform}
-+ */
-+function legacyAutolinkTransform(tree) {
-+  findAndReplace(
-+    tree,
-+    [
-+      [/(https?:\/\/|www(?=\.))([-.\w]+)([^ \t\r\n]*)/gi, findUrl],
-+      // [/([-.\w+]+)@([-\w]+(?:\.[-\w]+)+)/g, findEmail] # NOTE: original regex in 2.0.0
-+      [/(^|\s|\p{P}|\p{S})([-.\w+]+)@([-\w]+(?:\.[-\w]+)+)/gu, findEmailLegacy]
-+    ],
-+    {ignore: ['link', 'linkReference']}
-+  )
-+}
-+
-+/**
-+ * Helper function for legacy email matching
-+ * @param {string} _ - Unused parameter
-+ * @param {string} prefix - Email prefix
-+ * @param {string} name - Email name
-+ * @param {string} domain - Email domain
-+ * @returns {*} The processed email
-+ */
-+function findEmailLegacy(_, prefix, name, domain) {
-+  return findEmail(name + '@' + domain)
-+}
-+
- /**
-  * @type {ReplaceFunction}
-  * @param {string} _

pnpm-lock.yaml

@@ -4,11 +4,6 @@ settings:
   autoInstallPeers: true
   excludeLinksFromLockfile: false
 
-patchedDependencies:
-  mdast-util-gfm-autolink-literal@2.0.1:
-    hash: f63d515781110436299ab612306211a9621c6dfaec1ce1a19e2f27454dc70251
-    path: patches/mdast-util-gfm-autolink-literal@2.0.1.patch
-
 importers:
 
   .:
@@ -30,7 +25,7 @@ importers:
         version: 13.3.0
       next-i18next:
         specifier: 15.4.2
-        version: 15.4.2(i18next@23.16.8)(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
+        version: 15.4.2(i18next@23.16.8)(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
       prettier:
         specifier: 3.2.4
         version: 3.2.4
@@ -256,8 +251,8 @@ importers:
         specifier: 5.4.1
         version: 5.4.1
       pdfjs-dist:
-        specifier: 4.10.38
+        specifier: 4.4.168
-        version: 4.10.38
+        version: 4.4.168(encoding@0.1.13)
       pg:
         specifier: ^8.10.0
         version: 8.14.0
@@ -327,7 +322,7 @@ importers:
         version: 2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)
       '@chakra-ui/next-js':
         specifier: 2.4.2
-        version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
+        version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
       '@chakra-ui/react':
         specifier: 2.10.7
         version: 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -390,7 +385,7 @@ importers:
         version: 4.17.21
       next-i18next:
         specifier: 15.4.2
-        version: 15.4.2(i18next@23.16.8)(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
+        version: 15.4.2(i18next@23.16.8)(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
       papaparse:
         specifier: ^5.4.1
         version: 5.4.1
@@ -451,7 +446,7 @@ importers:
         version: 2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)
       '@chakra-ui/next-js':
         specifier: 2.4.2
-        version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
+        version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
       '@chakra-ui/react':
         specifier: 2.10.7
         version: 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -550,7 +545,7 @@ importers:
         version: 14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
       next-i18next:
         specifier: 15.4.2
-        version: 15.4.2(i18next@23.16.8)(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
+        version: 15.4.2(i18next@23.16.8)(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
       nprogress:
         specifier: ^0.2.0
         version: 0.2.0
@@ -2305,6 +2300,10 @@ packages:
     resolution: {integrity: sha512-9I2Zn6+NJLfaGoz9jN3lpwDgAYvfGeNYdbAIjJOqzs4Tpc+VU3Jqq4IofSUBKajiDS8k9fZIg18/z13mpk1bsA==}
     engines: {node: '>=8'}
 
+  '@mapbox/node-pre-gyp@1.0.11':
+    resolution: {integrity: sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==}
+    hasBin: true
+
   '@microsoft/tsdoc@0.15.1':
     resolution: {integrity: sha512-4aErSrCR/On/e5G2hDP0wjooqDdauzEbIq8hIkIe5pXV0rtWJZvdCEKL0ykZxex+IxIwBp0eGeV48hQN07dXtw==}
 
@@ -2354,70 +2353,6 @@ packages:
     cpu: [x64]
     os: [win32]
 
-  '@napi-rs/canvas-android-arm64@0.1.69':
-    resolution: {integrity: sha512-4icWTByY8zPvM9SelfQKf3I6kwXw0aI5drBOVrwfER5kjwXJd78FPSDSZkxDHjvIo9Q86ljl18Yr963ehA4sHQ==}
-    engines: {node: '>= 10'}
-    cpu: [arm64]
-    os: [android]
-
-  '@napi-rs/canvas-darwin-arm64@0.1.69':
-    resolution: {integrity: sha512-HOanhhYlHdukA+unjelT4Dg3ta7e820x87/AG2dKUMsUzH19jaeZs9bcYjzEy2vYi/dFWKz7cSv2yaIOudB8Yg==}
-    engines: {node: '>= 10'}
-    cpu: [arm64]
-    os: [darwin]
-
-  '@napi-rs/canvas-darwin-x64@0.1.69':
-    resolution: {integrity: sha512-SIp7WfhxAPnSVK9bkFfJp+84rbATCIq9jMUzDwpCLhQ+v+OqtXe4pggX1oeV+62/HK6BT1t18qRmJfyqwJ9f3g==}
-    engines: {node: '>= 10'}
-    cpu: [x64]
-    os: [darwin]
-
-  '@napi-rs/canvas-linux-arm-gnueabihf@0.1.69':
-    resolution: {integrity: sha512-Ls+KujCp6TGpkuMVFvrlx+CxtL+casdkrprFjqIuOAnB30Mct6bCEr+I83Tu29s3nNq4EzIGjdmA3fFAZG/Dtw==}
-    engines: {node: '>= 10'}
-    cpu: [arm]
-    os: [linux]
-
-  '@napi-rs/canvas-linux-arm64-gnu@0.1.69':
-    resolution: {integrity: sha512-m8VcGmeSBNRbHZBd1srvdM1aq/ScS2y8KqGqmCCEgJlytYK4jdULzAo2K/BPKE1v3xvn8oUPZDLI/NBJbJkEoA==}
-    engines: {node: '>= 10'}
-    cpu: [arm64]
-    os: [linux]
-
-  '@napi-rs/canvas-linux-arm64-musl@0.1.69':
-    resolution: {integrity: sha512-a3xjNRIeK2m2ZORGv2moBvv3vbkaFZG1QKMeiEv/BKij+rkztuEhTJGMar+buICFgS0fLgphXXsKNkUSJb7eRQ==}
-    engines: {node: '>= 10'}
-    cpu: [arm64]
-    os: [linux]
-
-  '@napi-rs/canvas-linux-riscv64-gnu@0.1.69':
-    resolution: {integrity: sha512-pClUoJF5wdC9AvD0mc15G9JffL1Q85nuH1rLSQPRkGmGmQOtRjw5E9xNbanz7oFUiPbjH7xcAXUjVAcf7tdgPQ==}
-    engines: {node: '>= 10'}
-    cpu: [riscv64]
-    os: [linux]
-
-  '@napi-rs/canvas-linux-x64-gnu@0.1.69':
-    resolution: {integrity: sha512-96X3bFAmzemfw84Ts6Jg/omL86uuynvK06MWGR/mp3JYNumY9RXofA14eF/kJIYelbYFWXcwpbcBR71lJ6G/YQ==}
-    engines: {node: '>= 10'}
-    cpu: [x64]
-    os: [linux]
-
-  '@napi-rs/canvas-linux-x64-musl@0.1.69':
-    resolution: {integrity: sha512-2QTsEFO72Kwkj53W9hc5y1FAUvdGx0V+pjJB+9oQF6Ys9+y989GyPIl5wZDzeh8nIJW6koZZ1eFa8pD+pA5BFQ==}
-    engines: {node: '>= 10'}
-    cpu: [x64]
-    os: [linux]
-
-  '@napi-rs/canvas-win32-x64-msvc@0.1.69':
-    resolution: {integrity: sha512-Q4YA8kVnKarApBVLu7F8icGlIfSll5Glswo5hY6gPS4Is2dCI8+ig9OeDM8RlwYevUIxKq8lZBypN8Q1iLAQ7w==}
-    engines: {node: '>= 10'}
-    cpu: [x64]
-    os: [win32]
-
-  '@napi-rs/canvas@0.1.69':
-    resolution: {integrity: sha512-ydvNeJMRm+l3T14yCoUKqjYQiEdXDq1isznI93LEBGYssXKfSaLNLHOkeM4z9Fnw9Pkt2EKOCAtW9cS4b00Zcg==}
-    engines: {node: '>= 10'}
-
   '@napi-rs/wasm-runtime@0.2.7':
     resolution: {integrity: sha512-5yximcFK5FNompXfJFoWanu5l8v1hNGqNHh9du1xETp9HWk/B/PzvchX55WYOPaIeNglG8++68AAiauBAtbnzw==}
 
@@ -3714,6 +3649,9 @@ packages:
   '@zilliz/milvus2-sdk-node@2.4.2':
     resolution: {integrity: sha512-fkPu7XXzfUvHoCnSPVOjqQpWuSnnn9x2NMmmCcIOyRzMeXIsrz4Mf/+M7LUzmT8J9F0Khx65B0rJgCu27YzWQw==}
 
+  abbrev@1.1.1:
+    resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==}
+
   abbrev@2.0.0:
     resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==}
     engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -3743,6 +3681,10 @@ packages:
     engines: {node: '>=0.4.0'}
     hasBin: true
 
+  agent-base@6.0.2:
+    resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
+    engines: {node: '>= 6.0.0'}
+
   agent-base@7.1.3:
     resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==}
     engines: {node: '>= 14'}
@@ -3850,6 +3792,14 @@ packages:
   append-field@1.0.0:
     resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==}
 
+  aproba@2.0.0:
+    resolution: {integrity: sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==}
+
+  are-we-there-yet@2.0.0:
+    resolution: {integrity: sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==}
+    engines: {node: '>=10'}
+    deprecated: This package is no longer supported.
+
   arg@4.1.3:
     resolution: {integrity: sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==}
 
@@ -4175,6 +4125,10 @@ packages:
   caniuse-lite@1.0.30001704:
     resolution: {integrity: sha512-+L2IgBbV6gXB4ETf0keSvLr7JUrRVbIaB/lrQ1+z8mRcQiisG5k+lG6O4n6Y5q6f5EuNfaYXKgymucphlEXQew==}
 
+  canvas@2.11.2:
+    resolution: {integrity: sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==}
+    engines: {node: '>=6'}
+
   ccount@2.0.1:
     resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==}
 
@@ -4366,6 +4320,10 @@ packages:
   color-string@1.9.1:
     resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==}
 
+  color-support@1.1.3:
+    resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==}
+    hasBin: true
+
   color2k@2.0.3:
     resolution: {integrity: sha512-zW190nQTIoXcGCaU08DvVNFTmQhUpnJfVuAKfWqUQkflXKpaDdpaYoM0iluLS9lgJNHyBF58KKA2FBEwkD7wog==}
 
@@ -4441,6 +4399,9 @@ packages:
   consola@2.15.3:
     resolution: {integrity: sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==}
 
+  console-control-strings@1.1.0:
+    resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==}
+
   content-disposition@0.5.4:
     resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
     engines: {node: '>= 0.6'}
@@ -4769,6 +4730,10 @@ packages:
   decode-named-character-reference@1.1.0:
     resolution: {integrity: sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w==}
 
+  decompress-response@4.2.1:
+    resolution: {integrity: sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==}
+    engines: {node: '>=8'}
+
   decompress-response@6.0.0:
     resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
     engines: {node: '>=10'}
@@ -4854,6 +4819,9 @@ packages:
     resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
     engines: {node: '>=0.4.0'}
 
+  delegates@1.0.0:
+    resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==}
+
   denque@2.1.0:
     resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
     engines: {node: '>=0.10'}
@@ -5563,6 +5531,11 @@ packages:
   functions-have-names@1.2.3:
     resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==}
 
+  gauge@3.0.2:
+    resolution: {integrity: sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==}
+    engines: {node: '>=10'}
+    deprecated: This package is no longer supported.
+
   generate-function@2.3.1:
     resolution: {integrity: sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==}
 
@@ -5715,6 +5688,9 @@ packages:
     resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==}
     engines: {node: '>= 0.4'}
 
+  has-unicode@2.0.1:
+    resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==}
+
   has-yarn@3.0.0:
     resolution: {integrity: sha512-IrsVwUHhEULx3R8f/aA8AHuEzAorplsab/v8HBzEiIukwq5i/EC+xmOW+HfP1OaDP+2JkgT1yILHN2O3UFIbcA==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
@@ -5802,6 +5778,10 @@ packages:
     resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==}
     engines: {node: '>=10.19.0'}
 
+  https-proxy-agent@5.0.1:
+    resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==}
+    engines: {node: '>= 6'}
+
   https-proxy-agent@7.0.6:
     resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==}
     engines: {node: '>= 14'}
@@ -6564,8 +6544,8 @@ packages:
   lexical@0.14.5:
     resolution: {integrity: sha512-ouV7Gyr9+3WT3WTrCgRAD3iZnlJWfs2/kBl2x3J2Q3X9uCWJn/zn21fQ8G1EUHlu0dvXPBmdk9hXb/FjTClt6Q==}
 
-  lib0@0.2.102:
+  lib0@0.2.99:
-    resolution: {integrity: sha512-g70kydI0I1sZU0ChO8mBbhw0oUW/8U0GHzygpvEIx8k+jgOpqnTSb/E+70toYVqHxBhrERD21TwD5QcZJQ40ZQ==}
+    resolution: {integrity: sha512-vwztYuUf1uf/1zQxfzRfO5yzfNKhTtgOByCruuiQQxWQXnPb8Itaube5ylofcV0oM0aKal9Mv+S1s1Ky0UYP1w==}
     engines: {node: '>=16'}
     hasBin: true
 
@@ -7054,6 +7034,10 @@ packages:
     resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==}
     engines: {node: '>=12'}
 
+  mimic-response@2.1.0:
+    resolution: {integrity: sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==}
+    engines: {node: '>=8'}
+
   mimic-response@3.1.0:
     resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
     engines: {node: '>=10'}
@@ -7231,6 +7215,9 @@ packages:
     resolution: {integrity: sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==}
     engines: {node: '>=12.0.0'}
 
+  nan@2.22.2:
+    resolution: {integrity: sha512-DANghxFkS1plDdRsX0X9pm0Z6SJNN6gBdtXfanwoZ8hooC5gosGFSBGRYHUVPz1asKA/kMRqDRdHrluZ61SpBQ==}
+
   nanoid@3.3.9:
     resolution: {integrity: sha512-SppoicMGpZvbF1l3z4x7No3OlIjP7QJvC9XR7AhZr1kL133KHnKPztkKDc+Ir4aJ/1VhTySrtKhrsycmrMQfvg==}
     engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
@@ -7363,6 +7350,11 @@ packages:
   non-layered-tidy-tree-layout@2.0.2:
     resolution: {integrity: sha512-gkXMxRzUH+PB0ax9dUN0yYF0S25BqeAYqhgMaLUFmpXLEk7Fcu8f4emJuOAY0V8kjDICxROIKsTAKsV/v355xw==}
 
+  nopt@5.0.0:
+    resolution: {integrity: sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==}
+    engines: {node: '>=6'}
+    hasBin: true
+
   nopt@7.2.1:
     resolution: {integrity: sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==}
     engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -7384,6 +7376,10 @@ packages:
     resolution: {integrity: sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==}
     engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
 
+  npmlog@5.0.1:
+    resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==}
+    deprecated: This package is no longer supported.
+
   nprogress@0.2.0:
     resolution: {integrity: sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==}
 
@@ -7612,6 +7608,10 @@ packages:
     resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==}
     engines: {node: '>=8'}
 
+  path2d@0.2.2:
+    resolution: {integrity: sha512-+vnG6S4dYcYxZd+CZxzXCNKdELYZSKfohrk98yajCo1PtRoDgCTrrwOvK1GT0UoAdVszagDVllQc0U1vaX4NUQ==}
+    engines: {node: '>=6'}
+
   pathe@1.1.2:
     resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==}
 
@@ -7625,9 +7625,9 @@ packages:
     resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==}
     engines: {node: '>= 14.16'}
 
-  pdfjs-dist@4.10.38:
+  pdfjs-dist@4.4.168:
-    resolution: {integrity: sha512-/Y3fcFrXEAsMjJXeL9J8+ZG9U01LbuWaYypvDW2ycW1jL269L3js3DVBjDJ0Up9Np1uqDXsDrRihHANhZOlwdQ==}
+    resolution: {integrity: sha512-MbkAjpwka/dMHaCfQ75RY1FXX3IewBVu6NGZOcxerRFlaBiIkZmUoR0jotX5VUzYZEXAGzSFtknWs5xRKliXPA==}
-    engines: {node: '>=20'}
+    engines: {node: '>=18'}
 
   peek-readable@5.4.2:
     resolution: {integrity: sha512-peBp3qZyuS6cNIJ2akRNG1uo1WJ1d0wTxg/fxMdZ0BqCVhx242bSFHM9eNqflfJVS9SsgkzgT/1UgnsurBOTMg==}
@@ -8464,6 +8464,9 @@ packages:
   simple-concat@1.0.1:
     resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
 
+  simple-get@3.1.1:
+    resolution: {integrity: sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==}
+
   simple-get@4.0.1:
     resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
 
@@ -9555,6 +9558,9 @@ packages:
     engines: {node: '>=8'}
     hasBin: true
 
+  wide-align@1.1.5:
+    resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==}
+
   widest-line@4.0.1:
     resolution: {integrity: sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==}
     engines: {node: '>=12'}
@@ -10740,7 +10746,7 @@ snapshots:
       '@chakra-ui/system': 2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1)
       react: 18.3.1
 
-  '@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
+  '@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
     dependencies:
       '@chakra-ui/react': 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
       '@emotion/cache': 11.14.0
@@ -11604,6 +11610,22 @@ snapshots:
 
   '@lukeed/ms@2.0.2': {}
 
+  '@mapbox/node-pre-gyp@1.0.11(encoding@0.1.13)':
+    dependencies:
+      detect-libc: 2.0.3
+      https-proxy-agent: 5.0.1
+      make-dir: 3.1.0
+      node-fetch: 2.7.0(encoding@0.1.13)
+      nopt: 5.0.0
+      npmlog: 5.0.1
+      rimraf: 3.0.2
+      semver: 7.7.1
+      tar: 6.2.1
+    transitivePeerDependencies:
+      - encoding
+      - supports-color
+    optional: true
+
   '@microsoft/tsdoc@0.15.1': {}
 
   '@mixmark-io/domino@2.2.0': {}
@@ -11641,50 +11663,6 @@ snapshots:
   '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3':
     optional: true
 
-  '@napi-rs/canvas-android-arm64@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas-darwin-arm64@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas-darwin-x64@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas-linux-arm-gnueabihf@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas-linux-arm64-gnu@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas-linux-arm64-musl@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas-linux-riscv64-gnu@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas-linux-x64-gnu@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas-linux-x64-musl@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas-win32-x64-msvc@0.1.69':
-    optional: true
-
-  '@napi-rs/canvas@0.1.69':
-    optionalDependencies:
-      '@napi-rs/canvas-android-arm64': 0.1.69
-      '@napi-rs/canvas-darwin-arm64': 0.1.69
-      '@napi-rs/canvas-darwin-x64': 0.1.69
-      '@napi-rs/canvas-linux-arm-gnueabihf': 0.1.69
-      '@napi-rs/canvas-linux-arm64-gnu': 0.1.69
-      '@napi-rs/canvas-linux-arm64-musl': 0.1.69
-      '@napi-rs/canvas-linux-riscv64-gnu': 0.1.69
-      '@napi-rs/canvas-linux-x64-gnu': 0.1.69
-      '@napi-rs/canvas-linux-x64-musl': 0.1.69
-      '@napi-rs/canvas-win32-x64-msvc': 0.1.69
-    optional: true
-
   '@napi-rs/wasm-runtime@0.2.7':
     dependencies:
       '@emnapi/core': 1.3.1
@@ -13088,6 +13066,9 @@ snapshots:
       protobufjs: 7.4.0
       winston: 3.17.0
 
+  abbrev@1.1.1:
+    optional: true
+
   abbrev@2.0.0: {}
 
   abort-controller@3.0.0:
@@ -13111,6 +13092,13 @@ snapshots:
 
   acorn@8.14.1: {}
 
+  agent-base@6.0.2:
+    dependencies:
+      debug: 4.4.0
+    transitivePeerDependencies:
+      - supports-color
+    optional: true
+
   agent-base@7.1.3: {}
 
   agentkeepalive@4.6.0:
@@ -13218,6 +13206,15 @@ snapshots:
 
   append-field@1.0.0: {}
 
+  aproba@2.0.0:
+    optional: true
+
+  are-we-there-yet@2.0.0:
+    dependencies:
+      delegates: 1.0.0
+      readable-stream: 3.6.2
+    optional: true
+
   arg@4.1.3: {}
 
   argparse@1.0.10:
@@ -13644,6 +13641,16 @@ snapshots:
 
   caniuse-lite@1.0.30001704: {}
 
+  canvas@2.11.2(encoding@0.1.13):
+    dependencies:
+      '@mapbox/node-pre-gyp': 1.0.11(encoding@0.1.13)
+      nan: 2.22.2
+      simple-get: 3.1.1
+    transitivePeerDependencies:
+      - encoding
+      - supports-color
+    optional: true
+
   ccount@2.0.1: {}
 
   chai@4.5.0:
@@ -13841,6 +13848,9 @@ snapshots:
       color-name: 1.1.4
       simple-swizzle: 0.2.2
 
+  color-support@1.1.3:
+    optional: true
+
   color2k@2.0.3: {}
 
   color@3.2.1:
@@ -13913,6 +13923,9 @@ snapshots:
 
   consola@2.15.3: {}
 
+  console-control-strings@1.1.0:
+    optional: true
+
   content-disposition@0.5.4:
     dependencies:
       safe-buffer: 5.2.1
@@ -14270,6 +14283,11 @@ snapshots:
     dependencies:
       character-entities: 2.0.2
 
+  decompress-response@4.2.1:
+    dependencies:
+      mimic-response: 2.1.0
+    optional: true
+
   decompress-response@6.0.0:
     dependencies:
       mimic-response: 3.1.0
@@ -14361,6 +14379,9 @@ snapshots:
 
   delayed-stream@1.0.0: {}
 
+  delegates@1.0.0:
+    optional: true
+
   denque@2.1.0: {}
 
   depd@2.0.0: {}
@@ -15322,6 +15343,19 @@ snapshots:
 
   functions-have-names@1.2.3: {}
 
+  gauge@3.0.2:
+    dependencies:
+      aproba: 2.0.0
+      color-support: 1.1.3
+      console-control-strings: 1.1.0
+      has-unicode: 2.0.1
+      object-assign: 4.1.1
+      signal-exit: 3.0.7
+      string-width: 4.2.3
+      strip-ansi: 6.0.1
+      wide-align: 1.1.5
+    optional: true
+
   generate-function@2.3.1:
     dependencies:
       is-property: 1.0.2
@@ -15486,6 +15520,9 @@ snapshots:
     dependencies:
       has-symbols: 1.1.0
 
+  has-unicode@2.0.1:
+    optional: true
+
   has-yarn@3.0.0: {}
 
   hasown@2.0.2:
@@ -15631,6 +15668,14 @@ snapshots:
       quick-lru: 5.1.1
       resolve-alpn: 1.2.1
 
+  https-proxy-agent@5.0.1:
+    dependencies:
+      agent-base: 6.0.2
+      debug: 4.4.0
+    transitivePeerDependencies:
+      - supports-color
+    optional: true
+
   https-proxy-agent@7.0.6:
     dependencies:
       agent-base: 7.1.3
@@ -16569,7 +16614,7 @@ snapshots:
 
   lexical@0.14.5: {}
 
-  lib0@0.2.102:
+  lib0@0.2.99:
     dependencies:
       isomorphic.js: 0.2.5
 
@@ -16842,7 +16887,7 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  mdast-util-gfm-autolink-literal@2.0.1(patch_hash=f63d515781110436299ab612306211a9621c6dfaec1ce1a19e2f27454dc70251):
+  mdast-util-gfm-autolink-literal@2.0.1:
     dependencies:
       '@types/mdast': 4.0.4
       ccount: 2.0.1
@@ -16890,7 +16935,7 @@ snapshots:
   mdast-util-gfm@3.1.0:
     dependencies:
       mdast-util-from-markdown: 2.0.2
-      mdast-util-gfm-autolink-literal: 2.0.1(patch_hash=f63d515781110436299ab612306211a9621c6dfaec1ce1a19e2f27454dc70251)
+      mdast-util-gfm-autolink-literal: 2.0.1
       mdast-util-gfm-footnote: 2.1.0
       mdast-util-gfm-strikethrough: 2.0.0
       mdast-util-gfm-table: 2.0.0
@@ -17397,6 +17442,9 @@ snapshots:
 
   mimic-fn@4.0.0: {}
 
+  mimic-response@2.1.0:
+    optional: true
+
   mimic-response@3.1.0: {}
 
   mimic-response@4.0.0: {}
@@ -17620,6 +17668,9 @@ snapshots:
     dependencies:
       lru-cache: 7.18.3
 
+  nan@2.22.2:
+    optional: true
+
   nanoid@3.3.9: {}
 
   nanoid@5.1.3: {}
@@ -17647,7 +17698,7 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
-  next-i18next@15.4.2(i18next@23.16.8)(next@14.2.26(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
+  next-i18next@15.4.2(i18next@23.16.8)(next@14.2.26(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
     dependencies:
       '@babel/runtime': 7.26.10
       '@types/hoist-non-react-statics': 3.3.6
@@ -17751,6 +17802,11 @@ snapshots:
 
   non-layered-tidy-tree-layout@2.0.2: {}
 
+  nopt@5.0.0:
+    dependencies:
+      abbrev: 1.1.1
+    optional: true
+
   nopt@7.2.1:
     dependencies:
       abbrev: 2.0.0
@@ -17767,6 +17823,14 @@ snapshots:
     dependencies:
       path-key: 4.0.0
 
+  npmlog@5.0.1:
+    dependencies:
+      are-we-there-yet: 2.0.0
+      console-control-strings: 1.1.0
+      gauge: 3.0.2
+      set-blocking: 2.0.0
+    optional: true
+
   nprogress@0.2.0: {}
 
   nth-check@2.1.1:
@@ -18051,6 +18115,9 @@ snapshots:
 
   path-type@4.0.0: {}
 
+  path2d@0.2.2:
+    optional: true
+
   pathe@1.1.2: {}
 
   pathe@2.0.3: {}
@@ -18059,9 +18126,13 @@ snapshots:
 
   pathval@2.0.0: {}
 
-  pdfjs-dist@4.10.38:
+  pdfjs-dist@4.4.168(encoding@0.1.13):
     optionalDependencies:
-      '@napi-rs/canvas': 0.1.69
+      canvas: 2.11.2(encoding@0.1.13)
+      path2d: 0.2.2
+    transitivePeerDependencies:
+      - encoding
+      - supports-color
 
   peek-readable@5.4.2: {}
 
@@ -19039,6 +19110,13 @@ snapshots:
 
   simple-concat@1.0.1: {}
 
+  simple-get@3.1.1:
+    dependencies:
+      decompress-response: 4.2.1
+      once: 1.4.0
+      simple-concat: 1.0.1
+    optional: true
+
   simple-get@4.0.1:
     dependencies:
       decompress-response: 6.0.0
@@ -20228,6 +20306,11 @@ snapshots:
       siginfo: 2.0.0
       stackback: 0.0.2
 
+  wide-align@1.1.5:
+    dependencies:
+      string-width: 4.2.3
+    optional: true
+
   widest-line@4.0.1:
     dependencies:
       string-width: 5.1.2
@@ -20357,7 +20440,7 @@ snapshots:
 
   yjs@13.6.24:
     dependencies:
-      lib0: 0.2.102
+      lib0: 0.2.99
 
   yn@3.1.1: {}
 

pnpm-workspace.yaml

@@ -1,7 +1,4 @@
 packages:
-  - packages/*
+  - 'packages/*'
-  - projects/*
+  - 'projects/*'
-  - scripts/icon
+  - 'scripts/icon'
-
-patchedDependencies:
-  mdast-util-gfm-autolink-literal@2.0.1: patches/mdast-util-gfm-autolink-literal@2.0.1.patch

projects/app/next.config.js

@@ -83,7 +83,6 @@ const nextConfig = {
     serverComponentsExternalPackages: [
       'mongoose',
       'pg',
-      'bullmq',
       '@zilliz/milvus2-sdk-node',
       "tiktoken",
     ],

projects/app/src/components/core/chat/ChatContainer/ChatBox/Input/VoiceInput.tsx

@@ -96,8 +96,6 @@ const MobileVoiceInput = ({
   const startYRef = useRef(0);
 
   const [isCancel, setIsCancel] = useState(false);
-  const canvasPosition = canvasRef.current?.getBoundingClientRect();
-  const maskBottom = canvasPosition ? `${window.innerHeight - canvasPosition.top}px` : '50px';
 
   const handleTouchStart = useCallback(
     (e: React.TouchEvent<HTMLDivElement>) => {
@@ -197,7 +195,7 @@ const MobileVoiceInput = ({
           position="fixed"
           left={0}
           right={0}
-          bottom={maskBottom}
+          bottom={'50px'}
           h={'200px'}
           bg="linear-gradient(to top, white, rgba(255, 255, 255, 0.7), rgba(255, 255, 255, 0))"
         >
@@ -297,7 +295,7 @@ const VoiceInput = forwardRef<VoiceInputComponentRef, VoiceInputProps>(
         }
       };
       startSpeak(finishWhisperTranscription);
-    }, [autoTTSResponse, onSendMessage, resetInputVal, startSpeak, whisperConfig?.autoSend]);
+    }, []);
 
     const onSpeach = useCallback(() => {
       if (isPc) {
@@ -305,7 +303,7 @@ const VoiceInput = forwardRef<VoiceInputComponentRef, VoiceInputProps>(
       } else {
         setMobilePreSpeak(true);
       }
-    }, [isPc, onStartSpeak]);
+    }, []);
     useImperativeHandle(ref, () => ({
       onSpeak: onSpeach
     }));

projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx

@@ -76,9 +76,8 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
   });
   const syncWebsite = async () => {
     await checkTeamWebSyncLimit();
-    postWebsiteSync({ datasetId: datasetId }).then(() => {
+    await postWebsiteSync({ datasetId: datasetId });
-      loadDatasetDetail(datasetId);
+    await loadDatasetDetail(datasetId);
-    });
   };
   const {
     isOpen: isOpenWebsiteModal,

projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx

@@ -33,11 +33,10 @@ import HeaderTagPopOver from './HeaderTagPopOver';
 import MyBox from '@fastgpt/web/components/common/MyBox';
 import Icon from '@fastgpt/web/components/common/Icon';
 import MyTag from '@fastgpt/web/components/common/Tag/index';
-import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 
 const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector'));
 
-const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
+const Header = ({}: {}) => {
   const { t } = useTranslation();
   const { feConfigs } = useSystemStore();
   const { isPc } = useSystem();
@@ -283,15 +282,13 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
                       >
                         {t('dataset:params_config')}
                       </Button>
-                      {!hasTrainingData && (
+                      <Button
-                        <Button
+                        variant={'whitePrimary'}
-                          variant={'whitePrimary'}
+                        onClick={openWebSyncConfirm}
-                          onClick={openWebSyncConfirm}
+                        leftIcon={<Icon name="common/confirm/restoreTip" w={'1rem'} />}
-                          leftIcon={<Icon name="common/confirm/restoreTip" w={'1rem'} />}
+                      >
-                        >
+                        {t('dataset:immediate_sync')}
-                          {t('dataset:immediate_sync')}
+                      </Button>
-                        </Button>
-                      )}
                     </HStack>
                   )}
                   {datasetDetail.status === DatasetStatusEnum.syncing && (
@@ -324,14 +321,6 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
                       {t('common:core.dataset.status.waiting')}
                     </MyTag>
                   )}
-                  {datasetDetail.status === DatasetStatusEnum.error && (
-                    <MyTag colorSchema="red" showDot px={3} h={'36px'}>
-                      <HStack spacing={1}>
-                        <Box>{t('dataset:status_error')}</Box>
-                        <QuestionTip color={'red.500'} label={datasetDetail.errorMsg} />
-                      </HStack>
-                    </MyTag>
-                  )}
                 </>
               ) : (
                 <Button

projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx

@@ -196,7 +196,7 @@ const CollectionCard = () => {
     <MyBox isLoading={isLoading} h={'100%'} py={[2, 4]}>
       <Flex ref={BoxRef} flexDirection={'column'} py={[1, 0]} h={'100%'} px={[2, 6]}>
         {/* header */}
-        <Header hasTrainingData={hasTrainingData} />
+        <Header />
 
         {/* collection table */}
         <TableContainer mt={3} overflowY={'auto'} fontSize={'sm'}>

projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx

@@ -115,7 +115,6 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
   const chunkSplitMode = watch('chunkSplitMode');
   const autoIndexes = watch('autoIndexes');
   const indexSize = watch('indexSize');
-  const imageIndex = watch('imageIndex');
 
   const trainingModeList = useMemo(() => {
     const list = Object.entries(DatasetCollectionDataProcessModeMap);
@@ -226,11 +225,7 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
           <HStack gap={[3, 7]}>
             <HStack flex={'1'} spacing={1}>
               <MyTooltip label={!feConfigs?.isPlus ? t('common:commercial_function_tip') : ''}>
-                <Checkbox
+                <Checkbox isDisabled={!feConfigs?.isPlus} {...register('autoIndexes')}>
-                  isDisabled={!feConfigs?.isPlus}
-                  isChecked={autoIndexes}
-                  {...register('autoIndexes')}
-                >
                   <FormLabel>{t('dataset:auto_indexes')}</FormLabel>
                 </Checkbox>
               </MyTooltip>
@@ -248,7 +243,6 @@ const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkForm
               >
                 <Checkbox
                   isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
-                  isChecked={imageIndex}
                   {...register('imageIndex')}
                 >
                   <FormLabel>{t('dataset:image_auto_parse')}</FormLabel>

projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx

@@ -20,14 +20,17 @@ import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import { shadowLight } from '@fastgpt/web/styles/theme';
 import CollectionChunkForm from '../../Form/CollectionChunkForm';
+import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
 
 function DataProcess() {
   const { t } = useTranslation();
   const { feConfigs } = useSystemStore();
 
-  const { goToNext, processParamsForm } = useContextSelector(DatasetImportContext, (v) => v);
+  const { goToNext, processParamsForm, chunkSize } = useContextSelector(
-  const { register, watch } = processParamsForm;
+    DatasetImportContext,
-  const customPdfParseValue = watch('customPdfParse');
+    (v) => v
+  );
+  const { register } = processParamsForm;
 
   const Title = useCallback(({ title }: { title: string }) => {
     return (
@@ -63,7 +66,7 @@ function DataProcess() {
                 >
                   {feConfigs.showCustomPdfParse && (
                     <HStack spacing={1}>
-                      <Checkbox isChecked={customPdfParseValue} {...register('customPdfParse')}>
+                      <Checkbox {...register('customPdfParse')}>
                         <FormLabel>{t('dataset:pdf_enhance_parse')}</FormLabel>
                       </Checkbox>
                       <QuestionTip label={t('dataset:pdf_enhance_parse_tips')} />

projects/app/src/pages/api/core/dataset/apiDataset/list.ts

@@ -1,6 +1,8 @@
+import { getFeishuAndYuqueDatasetFileList } from '@/service/core/dataset/apiDataset/controller';
 import { NextAPI } from '@/service/middleware/entry';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
+import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { useApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset/api';
 import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
@@ -12,6 +14,8 @@ export type GetApiDatasetFileListProps = {
   datasetId: string;
 };
 
+export type GetApiDatasetFileListResponse = APIFileItem[];
+
 async function handler(req: NextApiRequest) {
   let { searchKey = '', parentId = null, datasetId } = req.body;
 
@@ -31,7 +35,7 @@ async function handler(req: NextApiRequest) {
     return useApiDatasetRequest({ apiServer }).listFiles({ searchKey, parentId });
   }
   if (feishuServer || yuqueServer) {
-    return global.getProApiDatasetFileList({
+    return getFeishuAndYuqueDatasetFileList({
       feishuServer,
       yuqueServer,
       parentId

projects/app/src/pages/api/core/dataset/collection/read.ts

@@ -10,6 +10,7 @@ import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat';
 import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
 import { useApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset/api';
+import { POST } from '@fastgpt/service/common/api/plusRequest';
 
 export type readCollectionSourceQuery = {};
 
@@ -105,7 +106,8 @@ async function handler(
       }
 
       if (feishuServer || yuqueServer) {
-        return global.getProApiDatasetFilePreviewUrl({
+        return POST<string>(`/core/dataset/systemApiDataset`, {
+          type: 'read',
           apiFileId: collection.apiFileId,
           feishuServer,
           yuqueServer

projects/app/src/pages/api/core/dataset/detail.ts

@@ -30,21 +30,17 @@ async function handler(req: ApiRequestProps<Query>): Promise<DatasetItemType> {
     per: ReadPermissionVal
   });
 
-  const { status, errorMsg } = await (async () => {
+  const status = await (async () => {
     if (dataset.type === DatasetTypeEnum.websiteDataset) {
       return await getWebsiteSyncDatasetStatus(datasetId);
     }
 
-    return {
+    return DatasetStatusEnum.active;
-      status: DatasetStatusEnum.active,
-      errorMsg: undefined
-    };
   })();
 
   return {
     ...dataset,
     status,
-    errorMsg,
     apiServer: dataset.apiServer
       ? {
           baseUrl: dataset.apiServer.baseUrl,

projects/app/src/service/common/system/index.ts

@@ -11,18 +11,6 @@ import { defaultGroup, defaultTemplateTypes } from '@fastgpt/web/core/workflow/c
 import { MongoPluginGroups } from '@fastgpt/service/core/app/plugin/pluginGroupSchema';
 import { MongoTemplateTypes } from '@fastgpt/service/core/app/templates/templateTypeSchema';
 import { loadSystemModels } from '@fastgpt/service/core/ai/config/utils';
-import { POST } from '@fastgpt/service/common/api/plusRequest';
-import {
-  DeepRagSearchProps,
-  SearchDatasetDataResponse
-} from '@fastgpt/service/core/dataset/search/controller';
-import { AuthOpenApiLimitProps } from '@fastgpt/service/support/openapi/auth';
-import { ConcatUsageProps, CreateUsageProps } from '@fastgpt/global/support/wallet/usage/api';
-import {
-  getProApiDatasetFileContentRequest,
-  getProApiDatasetFileListRequest,
-  getProApiDatasetFilePreviewUrlRequest
-} from '@/service/core/dataset/apiDataset/controller';
 
 export const readConfigData = async (name: string) => {
   const splitName = name.split('.');
@@ -48,37 +36,12 @@ export const readConfigData = async (name: string) => {
 
 /* Init global variables */
 export function initGlobalVariables() {
-  function initPlusRequest() {
+  if (global.communityPlugins) return;
-    global.textCensorHandler = function textCensorHandler({ text }: { text: string }) {
-      return POST<{ code: number; message?: string }>('/common/censor/check', { text });
-    };
-
-    global.deepRagHandler = function deepRagHandler(data: DeepRagSearchProps) {
-      return POST<SearchDatasetDataResponse>('/core/dataset/deepRag', data);
-    };
-
-    global.authOpenApiHandler = function authOpenApiHandler(data: AuthOpenApiLimitProps) {
-      return POST<AuthOpenApiLimitProps>('/support/openapi/authLimit', data);
-    };
-
-    global.createUsageHandler = function createUsageHandler(data: CreateUsageProps) {
-      return POST('/support/wallet/usage/createUsage', data);
-    };
-
-    global.concatUsageHandler = function concatUsageHandler(data: ConcatUsageProps) {
-      return POST('/support/wallet/usage/concatUsage', data);
-    };
-
-    global.getProApiDatasetFileList = getProApiDatasetFileListRequest;
-    global.getProApiDatasetFileContent = getProApiDatasetFileContentRequest;
-    global.getProApiDatasetFilePreviewUrl = getProApiDatasetFilePreviewUrlRequest;
-  }
 
   global.communityPlugins = [];
   global.qaQueueLen = global.qaQueueLen ?? 0;
   global.vectorQueueLen = global.vectorQueueLen ?? 0;
   initHttpAgent();
-  initPlusRequest();
 }
 
 /* Init system data(Need to connected db). It only needs to run once */

projects/app/src/service/core/dataset/apiDataset/controller.ts

@@ -1,35 +1,14 @@
-import { APIFileItem, ApiFileReadContentResponse } from '@fastgpt/global/core/dataset/apiDataset';
+import { GetApiDatasetFileListResponse } from '@/pages/api/core/dataset/apiDataset/list';
+import { FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
 import { POST } from '@fastgpt/service/common/api/plusRequest';
-import {
-  GetProApiDatasetFileContentParams,
-  GetProApiDatasetFileListParams,
-  GetProApiDatasetFilePreviewUrlParams,
-  ProApiDatasetOperationTypeEnum
-} from '@fastgpt/service/core/dataset/apiDataset/proApi';
 
-export const getProApiDatasetFileListRequest = async (data: GetProApiDatasetFileListParams) => {
+export const getFeishuAndYuqueDatasetFileList = async (data: {
-  const res = await POST<APIFileItem[]>('/core/dataset/systemApiDataset', {
+  feishuServer?: FeishuServer;
-    type: ProApiDatasetOperationTypeEnum.LIST,
+  yuqueServer?: YuqueServer;
-    ...data
+  parentId?: string;
-  });
+}) => {
-  return res;
+  const res = await POST<GetApiDatasetFileListResponse>('/core/dataset/systemApiDataset', {
-};
+    type: 'list',
-
-export const getProApiDatasetFileContentRequest = async (
-  data: GetProApiDatasetFileContentParams
-) => {
-  const res = await POST<ApiFileReadContentResponse>('/core/dataset/systemApiDataset', {
-    type: ProApiDatasetOperationTypeEnum.CONTENT,
-    ...data
-  });
-  return res;
-};
-
-export const getProApiDatasetFilePreviewUrlRequest = async (
-  data: GetProApiDatasetFilePreviewUrlParams
-) => {
-  const res = await POST<string>('/core/dataset/systemApiDataset', {
-    type: ProApiDatasetOperationTypeEnum.READ,
     ...data
   });
   return res;

projects/app/src/service/core/dataset/data/controller.ts

@@ -17,7 +17,7 @@ import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
 import { countPromptTokens } from '@fastgpt/service/common/string/tiktoken';
 
 const formatIndexes = async ({
-  indexes = [],
+  indexes,
   q,
   a = '',
   indexSize,
@@ -66,6 +66,7 @@ const formatIndexes = async ({
     ];
   };
 
+  indexes = indexes || [];
   // If index not type, set it to custom
   indexes = indexes
     .map((item) => ({
@@ -92,7 +93,7 @@ const formatIndexes = async ({
   indexes = indexes.filter((item) => item.type !== DatasetDataIndexTypeEnum.default);
   indexes.push(...concatDefaultIndexes);
 
-  // Remove same text
+  // Filter same text
   indexes = indexes.filter(
     (item, index, self) => index === self.findIndex((t) => t.text === item.text)
   );
@@ -100,16 +101,12 @@ const formatIndexes = async ({
   const chekcIndexes = (
     await Promise.all(
       indexes.map(async (item) => {
-        if (item.type === DatasetDataIndexTypeEnum.default) {
-          return item;
-        }
-
         // If oversize tokens, split it
         const tokens = await countPromptTokens(item.text);
-        if (tokens > maxIndexSize) {
+        if (tokens > indexSize) {
           const splitText = splitText2Chunks({
             text: item.text,
-            chunkSize: indexSize,
+            chunkSize: 512,
             maxSize: maxIndexSize
           }).chunks;
           return splitText.map((text) => ({
@@ -117,7 +114,6 @@ const formatIndexes = async ({
             type: item.type
           }));
         }
-
         return item;
       })
     )
@@ -168,30 +164,24 @@ export async function insertData2Dataset({
   });
 
   // insert to vector store
-  const results: {
+  const result = await Promise.all(
-    tokens: number;
+    newIndexes.map(async (item) => {
-    index: {
+      const result = await insertDatasetDataVector({
-      dataId: string;
+        query: item.text,
-      type: `${DatasetDataIndexTypeEnum}`;
+        model: embModel,
-      text: string;
+        teamId,
-    };
+        datasetId,
-  }[] = [];
+        collectionId
-  for await (const item of newIndexes) {
+      });
-    const result = await insertDatasetDataVector({
+      return {
-      query: item.text,
+        tokens: result.tokens,
-      model: embModel,
+        index: {
-      teamId,
+          ...item,
-      datasetId,
+          dataId: result.insertId
-      collectionId
+        }
-    });
+      };
-    results.push({
+    })
-      tokens: result.tokens,
+  );
-      index: {
-        ...item,
-        dataId: result.insertId
-      }
-    });
-  }
 
   // 2. Create mongo data
   const [{ _id }] = await MongoDatasetData.create(
@@ -204,7 +194,7 @@ export async function insertData2Dataset({
         q,
         a,
         chunkIndex,
-        indexes: results.map((item) => item.index)
+        indexes: result.map((item) => item.index)
       }
     ],
     { session, ordered: true }
@@ -226,7 +216,7 @@ export async function insertData2Dataset({
 
   return {
     insertId: _id,
-    tokens: results.reduce((acc, cur) => acc + cur.tokens, 0)
+    tokens: result.reduce((acc, cur) => acc + cur.tokens, 0)
   };
 }
 
@@ -313,27 +303,25 @@ export async function updateData2Dataset({
   await mongoData.save();
 
   // 5. insert vector
-  const insertResults: {
+  const insertResult = await Promise.all(
-    tokens: number;
+    patchResult
-  }[] = [];
+      .filter((item) => item.type === 'create' || item.type === 'update')
-  for await (const item of patchResult) {
+      .map(async (item) => {
-    if (item.type === 'delete' || item.type === 'unChange') continue;
+        // insert new vector and update dateId
-
+        const result = await insertDatasetDataVector({
-    // insert new vector and update dateId
+          query: item.index.text,
-    const result = await insertDatasetDataVector({
+          model: getEmbeddingModel(model),
-      query: item.index.text,
+          teamId: mongoData.teamId,
-      model: getEmbeddingModel(model),
+          datasetId: mongoData.datasetId,
-      teamId: mongoData.teamId,
+          collectionId: mongoData.collectionId
-      datasetId: mongoData.datasetId,
+        });
-      collectionId: mongoData.collectionId
+        item.index.dataId = result.insertId;
-    });
+        return {
-    item.index.dataId = result.insertId;
+          tokens: result.tokens
-    insertResults.push({
+        };
-      tokens: result.tokens
+      })
-    });
+  );
-  }
+  const tokens = insertResult.reduce((acc, cur) => acc + cur.tokens, 0);
-
-  const tokens = insertResults.reduce((acc, cur) => acc + cur.tokens, 0);
 
   const newIndexes = patchResult
     .filter((item) => item.type !== 'delete')

projects/app/src/service/events/generateVector.ts

@@ -200,24 +200,19 @@ const rebuildData = async ({
 
   // update vector, update dataset_data rebuilding status, delete data from training
   // 1. Insert new vector to dataset_data
-  const updateResult: {
+  const updateResult = await Promise.all(
-    tokens: number;
+    mongoData.indexes.map(async (index, i) => {
-    insertId: string;
+      const result = await insertDatasetDataVector({
-  }[] = [];
+        query: index.text,
-  let i = 0;
+        model: getEmbeddingModel(trainingData.model),
-  for await (const index of mongoData.indexes) {
+        teamId: mongoData.teamId,
-    const result = await insertDatasetDataVector({
+        datasetId: mongoData.datasetId,
-      query: index.text,
+        collectionId: mongoData.collectionId
-      model: getEmbeddingModel(trainingData.model),
+      });
-      teamId: mongoData.teamId,
+      mongoData.indexes[i].dataId = result.insertId;
-      datasetId: mongoData.datasetId,
+      return result;
-      collectionId: mongoData.collectionId
+    })
-    });
+  );
-    mongoData.indexes[i].dataId = result.insertId;
-    updateResult.push(result);
-    i++;
-  }
-
   const { tokens } = await mongoSessionRun(async (session) => {
     // 2. Ensure that the training data is deleted after the Mongo update is successful
     await mongoData.save({ session });

projects/app/src/web/core/dataset/api.ts

@@ -52,7 +52,10 @@ import type {
 import type { UpdateDatasetDataProps } from '@fastgpt/global/core/dataset/controller';
 import type { DatasetFolderCreateBody } from '@/pages/api/core/dataset/folder/create';
 import type { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
-import type { GetApiDatasetFileListProps } from '@/pages/api/core/dataset/apiDataset/list';
+import type {
+  GetApiDatasetFileListProps,
+  GetApiDatasetFileListResponse
+} from '@/pages/api/core/dataset/apiDataset/list';
 import type {
   listExistIdQuery,
   listExistIdResponse
@@ -71,7 +74,6 @@ import type {
   getTrainingErrorBody,
   getTrainingErrorResponse
 } from '@/pages/api/core/dataset/training/getTrainingError';
-import type { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
 
 /* ======================== dataset ======================= */
 export const getDatasets = (data: GetDatasetListBody) =>
@@ -252,6 +254,6 @@ export const getCollectionSource = (data: readCollectionSourceBody) =>
 
 /* ================== apiDataset ======================== */
 export const getApiDatasetFileList = (data: GetApiDatasetFileListProps) =>
-  POST<APIFileItem[]>('/core/dataset/apiDataset/list', data);
+  POST<GetApiDatasetFileListResponse>('/core/dataset/apiDataset/list', data);
 export const getApiDatasetFileListExistId = (data: listExistIdQuery) =>
   GET<listExistIdResponse>('/core/dataset/apiDataset/listExistId', data);