perf: delete dataset (#3949)

* fix: collection list count

* fix: collection list count

* ai proxy ui

* perf: delete dataset

* perf: add dataset text index

* update doc

.github/workflows/fastgpt-preview-image.yml

@@ -68,14 +68,3 @@ jobs:
           SEALOS_TYPE: 'pr_comment'
           SEALOS_FILENAME: 'report.md'
           SEALOS_REPLACE_TAG: 'DEFAULT_REPLACE_DEPLOY'
-
-  helm-check:
-    runs-on: ubuntu-20.04
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Helm Check
-        run: |
-          helm dependency update files/helm/fastgpt
-          helm lint files/helm/fastgpt
-          helm package files/helm/fastgpt

.github/workflows/helm-release.yaml

@@ -24,6 +24,6 @@ jobs:
           export APP_VERSION=${{ steps.vars.outputs.tag }}
           export HELM_VERSION=${{ steps.vars.outputs.tag }}
           export HELM_REPO=ghcr.io/${{ github.repository_owner }}
-          helm dependency update files/helm/fastgpt
-          helm package files/helm/fastgpt --version ${HELM_VERSION}-helm --app-version ${APP_VERSION} -d bin
+          helm dependency update deploy/helm/fastgpt
+          helm package deploy/helm/fastgpt --version ${HELM_VERSION}-helm --app-version ${APP_VERSION} -d bin
           helm push bin/fastgpt-${HELM_VERSION}-helm.tgz oci://${HELM_REPO}

.vscode/nextapi.code-snippets

@@ -58,7 +58,7 @@
 		"body": [
 			"import '@/pages/api/__mocks__/base';",
 			"import { root } from '@/pages/api/__mocks__/db/init';",
-			"import { getTestRequest } from '@/test/utils';",
+			"import { getTestRequest } from '@fastgpt/service/test/utils'; ;",
 			"import { AppErrEnum } from '@fastgpt/global/common/error/code/app';",
 			"import handler from './demo';",
 			"",

deploy/docker/docker-compose-milvus.yml

@@ -114,15 +114,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.8.21-fix # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.21-fix # 阿里云
+    image: ghcr.io/labring/fastgpt-sandbox:v4.8.23-fix # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.23-fix # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.8.21-fix # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.21-fix # 阿里云
+    image: ghcr.io/labring/fastgpt:v4.8.23-fix # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.23-fix # 阿里云
     ports:
       - 3000:3000
     networks:
@@ -133,7 +133,7 @@ services:
       - sandbox
     restart: always
     environment:
-      # 前端访问地址: http://localhost:3000
+      # 前端外部可访问的地址，用于自动补全文件资源路径。例如 https:fastgpt.cn，不能填 localhost。这个值可以不填，不填则发给模型的图片会是一个相对路径，而不是全路径，模型可能伪造Host。
       - FE_DOMAIN=
       # root 密码，用户名为: root。如果需要修改 root 密码，直接修改这个环境变量，并重启即可。
       - DEFAULT_ROOT_PSW=1234

deploy/docker/docker-compose-pgvector.yml

@@ -72,15 +72,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.8.21-fix # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.21-fix # 阿里云
+    image: ghcr.io/labring/fastgpt-sandbox:v4.8.23-fix # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.23-fix # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.8.21-fix # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.21-fix # 阿里云
+    image: ghcr.io/labring/fastgpt:v4.8.23-fix # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.23-fix # 阿里云
     ports:
       - 3000:3000
     networks:
@@ -91,7 +91,7 @@ services:
       - sandbox
     restart: always
     environment:
-      # 前端访问地址: http://localhost:3000
+      # 前端外部可访问的地址，用于自动补全文件资源路径。例如 https:fastgpt.cn，不能填 localhost。这个值可以不填，不填则发给模型的图片会是一个相对路径，而不是全路径，模型可能伪造Host。
       - FE_DOMAIN=
       # root 密码，用户名为: root。如果需要修改 root 密码，直接修改这个环境变量，并重启即可。
       - DEFAULT_ROOT_PSW=1234

deploy/docker/docker-compose-zilliz.yml

@@ -53,15 +53,15 @@ services:
         wait $$!
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.8.21-fix # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.21-fix # 阿里云
+    image: ghcr.io/labring/fastgpt-sandbox:v4.8.23-fix # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.8.23-fix # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.8.21-fix # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.21-fix # 阿里云
+    image: ghcr.io/labring/fastgpt:v4.8.23-fix # git
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.8.23-fix # 阿里云
     ports:
       - 3000:3000
     networks:
@@ -71,7 +71,7 @@ services:
       - sandbox
     restart: always
     environment:
-      # 前端访问地址: http://localhost:3000
+      # 前端外部可访问的地址，用于自动补全文件资源路径。例如 https:fastgpt.cn，不能填 localhost。这个值可以不填，不填则发给模型的图片会是一个相对路径，而不是全路径，模型可能伪造Host。
       - FE_DOMAIN=
       # root 密码，用户名为: root。如果需要修改 root 密码，直接修改这个环境变量，并重启即可。
       - DEFAULT_ROOT_PSW=1234

docSite/content/zh-cn/docs/development/custom-models/bge-rerank.md

@@ -31,9 +31,9 @@ weight: 920
 
 3 个模型代码分别为：
 
-1. [https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-base](https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-base)
-2. [https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-large](https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-large)
-3. [https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-v2-m3](https://github.com/labring/FastGPT/tree/main/python/bge-rerank/bge-reranker-v2-m3)
+1. [https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-base](https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-base)
+2. [https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-large](https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-large)
+3. [https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-v2-m3](https://github.com/labring/FastGPT/tree/main/plugins/rerank-bge/bge-reranker-v2-m3)
 
 ### 3. 安装依赖
 

docSite/content/zh-cn/docs/development/custom-models/marker.md

@@ -19,7 +19,7 @@ PDF 是一个相对复杂的文件格式，在 FastGPT 内置的 pdf 解析器
 
 ### 1. 按照 Marker
 
-参考文档 [Marker 安装教程](https://github.com/labring/FastGPT/tree/main/python/pdf-marker)，安装 Marker 模型。封装的 API 已经适配了 FastGPT 自定义解析服务。
+参考文档 [Marker 安装教程](https://github.com/labring/FastGPT/tree/main/plugins/model/pdf-marker)，安装 Marker 模型。封装的 API 已经适配了 FastGPT 自定义解析服务。
 
 这里介绍快速 Docker 安装的方法：
 

docSite/content/zh-cn/docs/development/docker.md

@@ -118,7 +118,7 @@ brew install orbstack
 非 Linux 环境或无法访问外网环境，可手动创建一个目录，并下载配置文件和对应版本的`docker-compose.yml`，在这个文件夹中依据下载的配置文件运行docker，若作为本地开发使用推荐`docker-compose-pgvector`版本，并且自行拉取并运行`sandbox`和`fastgpt`，并在docker配置文件中注释掉`sandbox`和`fastgpt`的部分
 
 - [config.json](https://raw.githubusercontent.com/labring/FastGPT/refs/heads/main/projects/app/data/config.json)
-- [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/files/docker) (注意，不同向量库版本的文件不一样)
+- [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/deploy/docker) (注意，不同向量库版本的文件不一样)
 
 {{% alert icon="🤖" context="success" %}}
 
@@ -134,11 +134,11 @@ cd fastgpt
 curl -O https://raw.githubusercontent.com/labring/FastGPT/main/projects/app/data/config.json
 
 # pgvector 版本(测试推荐，简单快捷)
-curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/files/docker/docker-compose-pgvector.yml
+curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-pgvector.yml
 # milvus 版本
-# curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/files/docker/docker-compose-milvus.yml
+# curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-milvus.yml
 # zilliz 版本
-# curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/files/docker/docker-compose-zilliz.yml
+# curl -o docker-compose.yml https://raw.githubusercontent.com/labring/FastGPT/main/deploy/docker/docker-compose-zilliz.yml
 ```
 
 ### 2. 修改环境变量
@@ -201,6 +201,8 @@ docker restart oneapi
 
 在OneApi中添加合适的AI模型渠道。[点击查看相关教程](/docs/development/modelconfig/one-api/)
 
+只需要添加模型即可，模板已经配置好了oneapi的连接地址和令牌，无需变更。
+
 ### 5. 访问 FastGPT
 
 目前可以通过 `ip:3000` 直接访问(注意防火墙)。登录用户名为 `root`，密码为`docker-compose.yml`环境变量里设置的 `DEFAULT_ROOT_PSW`。
@@ -211,7 +213,7 @@ docker restart oneapi
 
 ### 6. 配置模型
 
-务必先配置至少一组模型，否则系统无法正常使用。
+登录FastGPT后，进入模型配置页面，务必先配置至少一个语言模型和一个向量模型，否则系统无法正常使用。
 
 [点击查看模型配置教程](/docs/development/modelConfig/intro/)
 

docSite/content/zh-cn/docs/development/faq.md

@@ -142,6 +142,10 @@ OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并
 3. ....
 
 
+### Tiktoken 下载失败
+
+由于 OneAPI 会在启动时从网络下载一个 tiktoken 的依赖，如果网络异常，就会导致启动失败。可以参考[OneAPI 离线部署](https://blog.csdn.net/wanh/article/details/139039216)解决。
+
 ## 四、常见模型问题
 
 ### 如何检查模型可用性问题

docSite/content/zh-cn/docs/development/modelConfig/intro.md

@@ -23,7 +23,7 @@ weight: 744
 
 {{% alert icon=" " context="info" %}}
 - [SiliconCloud(硅基流动)](https://cloud.siliconflow.cn/i/TR9Ym0c4): 提供开源模型调用的平台。
-- [Sealos AIProxy](https://hzh.sealos.run/?openapp=system-aiproxy): 提供国内各家模型代理，无需逐一申请 api。
+- [Sealos AIProxy](https://cloud.sealos.run/?uid=fnWRt09fZP&openapp=system-aiproxy): 提供国内各家模型代理，无需逐一申请 api。
 {{% /alert %}}
 
 在 OneAPI 配置好模型后，你就可以打开 FastGPT 页面，启用对应模型了。
@@ -467,4 +467,4 @@ OneAPI 的语言识别接口，无法正确的识别其他模型（会始终识
     "charsPointsPrice": 0
   }
 }
-```
+```

docSite/content/zh-cn/docs/development/openapi/chat.md

@@ -7,6 +7,12 @@ toc: true
 weight: 852
 ---
 
+# 如何获取 AppId
+
+可在应用详情的路径里获取 AppId。
+
+![](/imgs/appid.png)
+
 # 发起对话
 
 {{% alert icon="🤖 " context="success" %}}
@@ -102,8 +108,8 @@ curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
 {{% alert context="info" %}}
 - headers.Authorization: Bearer {{apikey}}
 - chatId: string | undefined 。
-  - 为 `undefined` 时（不传入），不使用 FastGpt 提供的上下文功能，完全通过传入的 messages 构建上下文。 不会将你的记录存储到数据库中，你也无法在记录汇总中查阅到。
-  - 为`非空字符串`时，意味着使用 chatId 进行对话，自动从 FastGpt 数据库取历史记录，并使用 messages 数组最后一个内容作为用户问题。请自行确保 chatId 唯一，长度小于250，通常可以是自己系统的对话框ID。
+  - 为 `undefined` 时（不传入），不使用 FastGpt 提供的上下文功能，完全通过传入的 messages 构建上下文。
+  - 为`非空字符串`时，意味着使用 chatId 进行对话，自动从 FastGpt 数据库取历史记录，并使用 messages 数组最后一个内容作为用户问题，其余 message 会被忽略。请自行确保 chatId 唯一，长度小于250，通常可以是自己系统的对话框ID。
 - messages: 结构与 [GPT接口](https://platform.openai.com/docs/api-reference/chat/object) chat模式一致。
 - responseChatItemId: string | undefined 。如果传入，则会将该值作为本次对话的响应消息的 ID，FastGPT 会自动将该 ID 存入数据库。请确保，在当前`chatId`下，`responseChatItemId`是唯一的。
 - detail: 是否返回中间值（模块状态，响应的完整结果等），`stream模式`下会通过`event`进行区分，`非stream模式`结果保存在`responseData`中。

docSite/content/zh-cn/docs/development/openapi/intro.md

@@ -1,6 +1,6 @@
 ---
-title: 'Api Key 使用与鉴权'
-description: 'FastGPT Api Key 使用与鉴权'
+title: 'OpenAPI 介绍'
+description: 'FastGPT OpenAPI 介绍'
 icon: 'key'
 draft: false
 toc: true
@@ -27,6 +27,7 @@ FastGPT 的 API Key **有 2 类**，一类是全局通用的 key (无法直接
 | --------------------- | --------------------- |
 | ![](/imgs/fastgpt-api2.jpg) | ![](/imgs/fastgpt-api1.jpg) |
 
+
 ## 基本配置
 
 OpenAPI 中，所有的接口都通过 Header.Authorization 进行鉴权。

docSite/content/zh-cn/docs/development/upgrading/482.md

@@ -20,7 +20,7 @@ SANDBOX_URL=内网地址
 
 ## Docker 部署
 
-可以拉取最新 [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/files/docker/docker-compose.yml) 文件参考
+可以拉取最新 [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/deploy/docker/docker-compose.yml) 文件参考
 
 1. 新增一个容器 `sandbox`
 2. fastgpt 和 fastgpt-pro(商业版) 容器新增环境变量: `SANDBOX_URL`

docSite/content/zh-cn/docs/development/upgrading/4822.md

@@ -1,5 +1,5 @@
 ---
-title: 'V4.8.22(进行中)'
+title: 'V4.8.22(包含升级脚本)'
 description: 'FastGPT V4.8.22 更新说明'
 icon: 'upgrade'
 draft: false
@@ -7,17 +7,55 @@ toc: true
 weight: 802
 ---
 
+## 🌟更新指南
 
-## 完整更新内容
+### 1. 做好数据库备份
 
-1. 新增 - AI 对话节点解析 <think></think> 标签内容，便于各类模型进行思考链输出。
-2. 优化 - 模型未配置时提示，减少冲突提示。
-3. 优化 - 使用记录代码。
-4. 修复 - 思考内容未进入到输出 Tokens.
-5. 修复 - 思考链流输出时，有时与正文顺序偏差。
-6. 修复 - API 调用工作流，如果传递的图片不支持 Head 检测时，图片会被过滤。已增加该类错误检测，避免被错误过滤。
-7. 修复 - 模板市场部分模板错误。
-8. 修复 - 免登录窗口无法正常判断语言识别是否开启。
-9. 修复 - 对话日志导出，未兼容 sub path。
-10. 修复 - list 接口在联查 member 时，存在空指针可能性。
-11. 修复 - 工作流基础节点无法升级。
+### 2. 更新镜像：
+
+- 更新 fastgpt 镜像 tag: v4.8.22
+- 更新 fastgpt-pro 商业版镜像 tag: v4.8.22
+- Sandbox 镜像无需更新
+
+### 3. 运行升级脚本
+
+仅商业版，并提供 Saas 服务的用户需要运行该升级脚本。
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv4822' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+会迁移联系方式到对应用户表中。
+
+## 🚀 新增内容
+
+1. AI 对话节点解析 `<think></think>` 标签内容作为思考链，便于各类模型进行思考链输出。需主动开启模型输出思考。
+2. 对话 API 优化，无论是否传递 chatId，都会保存对话日志。未传递 chatId，则随机生成一个 chatId 来进行存储。
+3. ppio 模型提供商
+
+## ⚙️ 优化
+
+1. 模型未配置时提示，减少冲突提示。
+2. 使用记录代码。
+3. 内容提取节点，字段描述过长时换行。同时修改其输出名用 key，而不是 description。
+4. 团队管理交互。
+5. 对话接口，非流响应，增加报错字段。
+
+## 🐛 修复
+
+1. 思考内容未进入到输出 Tokens.
+2. 思考链流输出时，有时与正文顺序偏差。
+3. API 调用工作流，如果传递的图片不支持 Head 检测时，图片会被过滤。已增加该类错误检测，避免被错误过滤。
+4. 模板市场部分模板错误。
+5. 免登录窗口无法正常判断语言识别是否开启。
+6. 对话日志导出，未兼容 sub path。
+7. 切换团队时未刷新成员列表
+8. list 接口在联查 member 时，存在空指针可能性。 
+9. 工作流基础节点无法升级。
+10. 向量检索结果未去重。
+11. 用户选择节点无法正常连线。
+12. 对话记录保存时，source 未正常记录。

docSite/content/zh-cn/docs/development/upgrading/4823.md

@@ -0,0 +1,54 @@
+---
+title: 'V4.8.23'
+description: 'FastGPT V4.8.23 更新说明'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 802
+---
+
+## 更新指南
+
+### 1. 做好数据库备份
+
+### 2. 更新镜像：
+
+- 更新 fastgpt 镜像 tag: v4.8.23-fix
+- 更新 fastgpt-pro 商业版镜像 tag: v4.8.23-fix
+- Sandbox 镜像无需更新
+
+### 3. 运行升级脚本
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv4823' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+脚本会清理一些知识库脏数据，主要是多余的全文索引。
+
+## 🚀 新增内容
+
+1. 增加默认“知识库文本理解模型”配置
+2. AI proxy V1版，可替换 OneAPI使用，同时提供完整模型调用日志，便于排查问题。
+3. 增加工单入口支持。
+
+## ⚙️ 优化
+
+1. 模型配置表单，增加必填项校验。
+2. 集合列表数据统计方式，提高大数据量统计性能。
+3. 优化数学公式，转义 Latex 格式成 Markdown 格式。
+4. 解析文档图片，图片太大时，自动忽略。
+5. 时间选择器，当天开始时间自动设0，结束设置设 23:59:59，避免 UI 与实际逻辑偏差。
+6. 升级 mongoose 库版本依赖。
+
+## 🐛 修复
+
+1. 标签过滤时，子文件夹未成功过滤。
+2. 暂时移除 md 阅读优化，避免链接分割错误。
+3. 离开团队时，未刷新成员列表。
+4. PPTX 编码错误，导致解析失败。
+5. 删除知识库单条数据时，全文索引未跟随删除。
+6. 修复 Mongo Dataset text 索引在查询数据时未生效。

docSite/content/zh-cn/docs/development/upgrading/483.md

@@ -15,7 +15,7 @@ weight: 821
 
 ## V4.8.3 更新说明
 
-1. 新增 - 支持 Milvus 数据库， 可参考最新的 [docker-compose-milvus.yml](https://github.com/labring/FastGPT/blob/main/files/docker/docker-compose-milvus.yml).
+1. 新增 - 支持 Milvus 数据库， 可参考最新的 [docker-compose-milvus.yml](https://github.com/labring/FastGPT/blob/main/deploy/docker/docker-compose-milvus.yml).
 2. 新增 - 给 chat 接口 empty answer 增加 log，便于排查模型问题。
 3. 新增 - ifelse判断器，字符串支持正则。
 4. 新增 - 代码运行支持 console.log 输出调试。

docSite/content/zh-cn/docs/guide/course/quick-start.md

@@ -7,11 +7,11 @@ toc: true
 weight: 102
 ---
 
-更多使用技巧，[查看视屏教程](https://www.bilibili.com/video/BV1sH4y1T7s9)
+更多使用技巧，[查看视频教程](https://www.bilibili.com/video/BV1sH4y1T7s9)
 
 ## 知识库
 
-开始前，请准备一份测试电子文档，WORD，PDF，TXT，excel，markdown 都可以，比如公司休假制度，不涉密的销售说辞，产品知识等等。
+开始前，请准备一份测试电子文档，WORD、PDF、TXT、excel、markdown 都可以，比如公司休假制度、不涉密的销售说辞、产品知识等等。
 
 这里使用 FastGPT 中文 README 文件为例。
 
@@ -31,7 +31,7 @@ weight: 102
 
 ![](/imgs/upload-data.png)
 
-点击上传后我们需要等待数据处理完成，等到我们上传的文件状态为可用。
+点击上传后我们需要等待数据处理完成，直到我们上传的文件状态为可用。
 
 ![](/imgs/upload-data2.png)
 

packages/README.md

@@ -0,0 +1,3 @@
+# 目录说明
+
+该目录为 FastGPT 的依赖包，多端复用。

packages/global/common/error/code/dataset.ts

@@ -4,6 +4,7 @@ import { ErrType } from '../errorCode';
 /* dataset: 501000 */
 export enum DatasetErrEnum {
   unExist = 'unExistDataset',
+  unExistCollection = 'unExistCollection',
   unAuthDataset = 'unAuthDataset',
   unCreateCollection = 'unCreateCollection',
   unAuthDatasetCollection = 'unAuthDatasetCollection',
@@ -28,6 +29,10 @@ const datasetErr = [
     statusText: DatasetErrEnum.unExist,
     message: 'core.dataset.error.unExistDataset'
   },
+  {
+    statusText: DatasetErrEnum.unExistCollection,
+    message: i18nT('common:error_collection_not_exist')
+  },
   {
     statusText: DatasetErrEnum.unAuthDataset,
     message: 'core.dataset.error.unAuthDataset'

packages/global/common/string/time.ts

@@ -7,12 +7,14 @@ import { i18nT } from '../../../web/i18n/utils';
 dayjs.extend(utc);
 dayjs.extend(timezone);
 
-export const formatTime2YMDHMW = (time?: Date) => dayjs(time).format('YYYY-MM-DD HH:mm:ss dddd');
-export const formatTime2YMDHMS = (time?: Date) =>
+export const formatTime2YMDHMW = (time?: Date | number) =>
+  dayjs(time).format('YYYY-MM-DD HH:mm:ss dddd');
+export const formatTime2YMDHMS = (time?: Date | number) =>
   time ? dayjs(time).format('YYYY-MM-DD HH:mm:ss') : '';
-export const formatTime2YMDHM = (time?: Date) =>
+export const formatTime2YMDHM = (time?: Date | number) =>
   time ? dayjs(time).format('YYYY-MM-DD HH:mm') : '';
-export const formatTime2YMD = (time?: Date) => (time ? dayjs(time).format('YYYY-MM-DD') : '');
+export const formatTime2YMD = (time?: Date | number) =>
+  time ? dayjs(time).format('YYYY-MM-DD') : '';
 export const formatTime2HM = (time: Date = new Date()) => dayjs(time).format('HH:mm');
 
 /**

packages/global/common/system/types/index.d.ts

@@ -41,6 +41,7 @@ export type FastGPTConfigFileType = {
 };
 
 export type FastGPTFeConfigsType = {
+  show_workorder?: boolean;
   show_emptyChat?: boolean;
   register_method?: ['email' | 'phone' | 'sync'];
   login_method?: ['email' | 'phone']; // Attention: login method is diffrent with oauth
@@ -53,6 +54,7 @@ export type FastGPTFeConfigsType = {
   show_promotion?: boolean;
   show_team_chat?: boolean;
   show_compliance_copywriting?: boolean;
+  show_aiproxy?: boolean;
   concatMd?: string;
 
   docUrl?: string;

packages/global/core/ai/model.d.ts

@@ -17,6 +17,8 @@ type BaseModelItemType = {
   isActive?: boolean;
   isCustom?: boolean;
   isDefault?: boolean;
+  isDefaultDatasetTextModel?: boolean;
+  isDefaultDatasetImageModel?: boolean;
 
   // If has requestUrl, it will request the model directly
   requestUrl?: string;

packages/global/core/ai/provider.ts

@@ -72,11 +72,6 @@ export const ModelProviderList: ModelProviderType[] = [
     name: 'Groq',
     avatar: 'model/groq'
   },
-  {
-    id: 'AliCloud',
-    name: i18nT('common:model_alicloud'),
-    avatar: 'model/alicloud'
-  },
   {
     id: 'Qwen',
     name: i18nT('common:model_qwen'),
@@ -87,6 +82,11 @@ export const ModelProviderList: ModelProviderType[] = [
     name: i18nT('common:model_doubao'),
     avatar: 'model/doubao'
   },
+  {
+    id: 'DeepSeek',
+    name: 'DeepSeek',
+    avatar: 'model/deepseek'
+  },
   {
     id: 'ChatGLM',
     name: i18nT('common:model_chatglm'),
@@ -97,11 +97,6 @@ export const ModelProviderList: ModelProviderType[] = [
     name: i18nT('common:model_ernie'),
     avatar: 'model/ernie'
   },
-  {
-    id: 'DeepSeek',
-    name: 'DeepSeek',
-    avatar: 'model/deepseek'
-  },
   {
     id: 'Moonshot',
     name: i18nT('common:model_moonshot'),
@@ -163,6 +158,11 @@ export const ModelProviderList: ModelProviderType[] = [
     name: i18nT('common:model_moka'),
     avatar: 'model/moka'
   },
+  {
+    id: 'AliCloud',
+    name: i18nT('common:model_alicloud'),
+    avatar: 'model/alicloud'
+  },
   {
     id: 'Siliconflow',
     name: i18nT('common:model_siliconflow'),

packages/global/core/dataset/type.d.ts

@@ -192,6 +192,7 @@ export type DatasetCollectionItemType = CollectionWithDatasetType & {
   sourceId?: string;
   file?: DatasetFileSchema;
   permission: DatasetPermission;
+  indexAmount: number;
 };
 
 /* ================= data ===================== */

packages/global/core/workflow/runtime/utils.ts

@@ -420,137 +420,3 @@ export function rewriteNodeOutputByHistories(
     };
   });
 }
-
-// Parse <think></think> tags to think and answer - unstream response
-export const parseReasoningContent = (text: string): [string, string] => {
-  const regex = /<think>([\s\S]*?)<\/think>/;
-  const match = text.match(regex);
-
-  if (!match) {
-    return ['', text];
-  }
-
-  const thinkContent = match[1].trim();
-
-  // Add answer (remaining text after think tag)
-  const answerContent = text.slice(match.index! + match[0].length);
-
-  return [thinkContent, answerContent];
-};
-
-// Parse <think></think> tags to think and answer - stream response
-export const parseReasoningStreamContent = () => {
-  let isInThinkTag: boolean | undefined;
-
-  const startTag = '<think>';
-  let startTagBuffer = '';
-
-  const endTag = '</think>';
-  let endTagBuffer = '';
-
-  /* 
-    parseReasoning - 只控制是否主动解析 <think></think>，如果接口已经解析了，仍然会返回 think 内容。
-  */
-  const parsePart = (
-    part: {
-      choices: {
-        delta: {
-          content?: string;
-          reasoning_content?: string;
-        };
-      }[];
-    },
-    parseReasoning = false
-  ): [string, string] => {
-    const content = part.choices?.[0]?.delta?.content || '';
-
-    // @ts-ignore
-    const reasoningContent = part.choices?.[0]?.delta?.reasoning_content || '';
-    if (reasoningContent || !parseReasoning) {
-      isInThinkTag = false;
-      return [reasoningContent, content];
-    }
-
-    if (!content) {
-      return ['', ''];
-    }
-
-    // 如果不在 think 标签中，或者有 reasoningContent(接口已解析），则返回 reasoningContent 和 content
-    if (isInThinkTag === false) {
-      return ['', content];
-    }
-
-    // 检测是否为 think 标签开头的数据
-    if (isInThinkTag === undefined) {
-      // Parse content think and answer
-      startTagBuffer += content;
-      // 太少内容时候，暂时不解析
-      if (startTagBuffer.length < startTag.length) {
-        return ['', ''];
-      }
-
-      if (startTagBuffer.startsWith(startTag)) {
-        isInThinkTag = true;
-        return [startTagBuffer.slice(startTag.length), ''];
-      }
-
-      // 如果未命中 think 标签，则认为不在 think 标签中，返回 buffer 内容作为 content
-      isInThinkTag = false;
-      return ['', startTagBuffer];
-    }
-
-    // 确认是 think 标签内容，开始返回 think 内容，并实时检测 </think>
-    /* 
-      检测 </think> 方案。
-      存储所有疑似 </think> 的内容，直到检测到完整的 </think> 标签或超出 </think> 长度。
-      content 返回值包含以下几种情况:
-        abc - 完全未命中尾标签
-        abc<th - 命中一部分尾标签
-        abc</think> - 完全命中尾标签
-        abc</think>abc - 完全命中尾标签
-        </think>abc - 完全命中尾标签
-        k>abc - 命中一部分尾标签
-    */
-    // endTagBuffer 专门用来记录疑似尾标签的内容
-    if (endTagBuffer) {
-      endTagBuffer += content;
-      if (endTagBuffer.includes(endTag)) {
-        isInThinkTag = false;
-        const answer = endTagBuffer.slice(endTag.length);
-        return ['', answer];
-      } else if (endTagBuffer.length >= endTag.length) {
-        // 缓存内容超出尾标签长度，且仍未命中 </think>，则认为本次猜测 </think> 失败，仍处于 think 阶段。
-        const tmp = endTagBuffer;
-        endTagBuffer = '';
-        return [tmp, ''];
-      }
-      return ['', ''];
-    } else if (content.includes(endTag)) {
-      // 返回内容，完整命中</think>，直接结束
-      isInThinkTag = false;
-      const [think, answer] = content.split(endTag);
-      return [think, answer];
-    } else {
-      // 无 buffer，且未命中 </think>，开始疑似 </think> 检测。
-      for (let i = 1; i < endTag.length; i++) {
-        const partialEndTag = endTag.slice(0, i);
-        // 命中一部分尾标签
-        if (content.endsWith(partialEndTag)) {
-          const think = content.slice(0, -partialEndTag.length);
-          endTagBuffer += partialEndTag;
-          return [think, ''];
-        }
-      }
-    }
-
-    // 完全未命中尾标签，还是 think 阶段。
-    return [content, ''];
-  };
-
-  const getStartTagBuffer = () => startTagBuffer;
-
-  return {
-    parsePart,
-    getStartTagBuffer
-  };
-};

packages/global/support/user/api.d.ts

@@ -1,5 +1,9 @@
+import { MemberGroupSchemaType, MemberGroupType } from 'support/permission/memberGroup/type';
 import { OAuthEnum } from './constant';
 import { TrackRegisterParams } from './login/api';
+import { TeamMemberStatusEnum } from './team/constant';
+import { OrgType } from './team/org/type';
+import { TeamMemberItemType } from './team/type';
 
 export type PostLoginProps = {
   username: string;
@@ -21,3 +25,9 @@ export type FastLoginProps = {
   token: string;
   code: string;
 };
+
+export type SearchResult = {
+  members: Omit<TeamMemberItemType, 'teamId' | 'permission'>[];
+  orgs: Omit<OrgType, 'permission' | 'members'>[];
+  groups: MemberGroupSchemaType[];
+};

packages/global/support/user/team/controller.d.ts

@@ -13,6 +13,7 @@ export type CreateTeamProps = {
   defaultTeam?: boolean;
   memberName?: string;
   memberAvatar?: string;
+  notificationAccount?: string;
 };
 export type UpdateTeamProps = Omit<ThirdPartyAccountType, 'externalWorkflowVariable'> & {
   name?: string;
@@ -39,6 +40,12 @@ export type UpdateInviteProps = {
   tmbId: string;
   status: TeamMemberSchema['status'];
 };
+
+export type UpdateStatusProps = {
+  tmbId: string;
+  status: TeamMemberSchema['status'];
+};
+
 export type InviteMemberResponse = Record<
   'invite' | 'inValid' | 'inTeam',
   { username: string; userId: string }[]

packages/global/support/user/team/type.d.ts

@@ -34,6 +34,7 @@ export type TeamTagSchema = TeamTagItemType & {
   _id: string;
   teamId: string;
   createTime: Date;
+  updateTime?: Date;
 };
 
 export type TeamMemberSchema = {
@@ -41,6 +42,7 @@ export type TeamMemberSchema = {
   teamId: string;
   userId: string;
   createTime: Date;
+  updateTime?: Date;
   name: string;
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;
@@ -79,6 +81,9 @@ export type TeamMemberItemType = {
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;
   permission: TeamPermission;
+  contact?: string;
+  createTime: Date;
+  updateTime?: Date;
 };
 
 export type TeamTagItemType = {

packages/global/support/user/type.d.ts

@@ -17,6 +17,7 @@ export type UserModelSchema = {
   fastgpt_sem?: {
     keyword: string;
   };
+  contact?: string;
 };
 
 export type UserType = {
@@ -26,9 +27,9 @@ export type UserType = {
   timezone: string;
   promotionRate: UserModelSchema['promotionRate'];
   team: TeamTmbItemType;
-  standardInfo?: standardInfoType;
   notificationAccount?: string;
   permission: TeamPermission;
+  contact?: string;
 };
 
 export type SourceMemberType = {

packages/service/common/file/csv.ts

@@ -0,0 +1,4 @@
+export const generateCsv = (headers: string[], data: string[][]) => {
+  const csv = [headers.join(','), ...data.map((row) => row.join(','))].join('\n');
+  return csv;
+};

packages/service/common/file/gridfs/controller.ts

@@ -18,10 +18,10 @@ export function getGFSCollection(bucket: `${BucketNameEnum}`) {
   MongoDatasetFileSchema;
   MongoChatFileSchema;
 
-  return connectionMongo.connection.db.collection(`${bucket}.files`);
+  return connectionMongo.connection.db!.collection(`${bucket}.files`);
 }
 export function getGridBucket(bucket: `${BucketNameEnum}`) {
-  return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db, {
+  return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db!, {
     bucketName: bucket,
     // @ts-ignore
     readPreference: ReadPreference.SECONDARY_PREFERRED // Read from secondary node

packages/service/common/file/image/controller.ts

@@ -118,7 +118,7 @@ export async function delImgByRelatedId({
 }: {
   teamId: string;
   relateIds: string[];
-  session: ClientSession;
+  session?: ClientSession;
 }) {
   if (relateIds.length === 0) return;
 

packages/service/common/file/read/utils.ts

@@ -111,15 +111,21 @@ export const readRawContentByFileBuffer = async ({
   // markdown data format
   if (imageList) {
     await batchRun(imageList, async (item) => {
-      const src = await uploadMongoImg({
-        base64Img: `data:${item.mime};base64,${item.base64}`,
-        teamId,
-        // expiredTime: addHours(new Date(), 1),
-        metadata: {
-          ...metadata,
-          mime: item.mime
+      const src = await (async () => {
+        try {
+          return await uploadMongoImg({
+            base64Img: `data:${item.mime};base64,${item.base64}`,
+            teamId,
+            // expiredTime: addHours(new Date(), 1),
+            metadata: {
+              ...metadata,
+              mime: item.mime
+            }
+          });
+        } catch (error) {
+          return '';
         }
-      });
+      })();
       rawText = rawText.replace(item.uuid, src);
       if (formatText) {
         formatText = formatText.replace(item.uuid, src);

packages/service/common/mongo/index.ts

@@ -38,10 +38,12 @@ const addCommonMiddleware = (schema: mongoose.Schema) => {
     schema.post(op, function (this: any, result: any, next) {
       if (this._startTime) {
         const duration = Date.now() - this._startTime;
-
         const warnLogData = {
-          query: this._query,
-          op,
+          collectionName: this.collection?.name,
+          op: this.op,
+          ...(this._query && { query: this._query }),
+          ...(this._update && { update: this._update }),
+          ...(this._delete && { delete: this._delete }),
           duration
         };
 

packages/service/common/mongo/init.ts

@@ -16,16 +16,30 @@ export async function connectMongo(): Promise<Mongoose> {
 
   console.log('mongo start connect');
   try {
-    connectionMongo.set('strictQuery', true);
+    // Remove existing listeners to prevent duplicates
+    connectionMongo.connection.removeAllListeners('error');
+    connectionMongo.connection.removeAllListeners('disconnected');
+    connectionMongo.set('strictQuery', false);
 
     connectionMongo.connection.on('error', async (error) => {
       console.log('mongo error', error);
-      await connectionMongo.disconnect();
-      await delay(1000);
-      connectMongo();
+      try {
+        if (connectionMongo.connection.readyState !== 0) {
+          await connectionMongo.disconnect();
+          await delay(1000);
+          await connectMongo();
+        }
+      } catch (error) {}
     });
-    connectionMongo.connection.on('disconnected', () => {
+    connectionMongo.connection.on('disconnected', async () => {
       console.log('mongo disconnected');
+      try {
+        if (connectionMongo.connection.readyState !== 0) {
+          await connectionMongo.disconnect();
+          await delay(1000);
+          await connectMongo();
+        }
+      } catch (error) {}
     });
 
     await connectionMongo.connect(process.env.MONGODB_URI as string, {

packages/service/common/system/constants.ts

@@ -1 +1,4 @@
 export const FastGPTProUrl = process.env.PRO_URL ? `${process.env.PRO_URL}/api` : '';
+export const isFastGPTMainService = !!process.env.PRO_URL;
+// @ts-ignore
+export const isFastGPTProService = () => !!global.systemConfig;

packages/service/common/vectorStore/controller.ts

@@ -21,6 +21,7 @@ export const recallFromVectorStore = Vector.embRecall;
 export const getVectorDataByTime = Vector.getVectorDataByTime;
 export const getVectorCountByTeamId = Vector.getVectorCountByTeamId;
 export const getVectorCountByDatasetId = Vector.getVectorCountByDatasetId;
+export const getVectorCountByCollectionId = Vector.getVectorCountByCollectionId;
 
 export const insertDatasetDataVector = async ({
   model,

packages/service/common/vectorStore/milvus/class.ts

@@ -321,6 +321,23 @@ export class MilvusCtrl {
 
     return total;
   };
+  getVectorCountByCollectionId = async (
+    teamId: string,
+    datasetId: string,
+    collectionId: string
+  ) => {
+    const client = await this.getClient();
+
+    const result = await client.query({
+      collection_name: DatasetVectorTableName,
+      output_fields: ['count(*)'],
+      filter: `(teamId == "${String(teamId)}") and (datasetId == "${String(datasetId)}") and (collectionId == "${String(collectionId)}")`
+    });
+
+    const total = result.data?.[0]?.['count(*)'] as number;
+
+    return total;
+  };
 
   getVectorDataByTime = async (start: Date, end: Date) => {
     const client = await this.getClient();

packages/service/common/vectorStore/pg/class.ts

@@ -240,6 +240,23 @@ export class PgVectorCtrl {
       where: [['team_id', String(teamId)], 'and', ['dataset_id', String(datasetId)]]
     });
 
+    return total;
+  };
+  getVectorCountByCollectionId = async (
+    teamId: string,
+    datasetId: string,
+    collectionId: string
+  ) => {
+    const total = await PgClient.count(DatasetVectorTableName, {
+      where: [
+        ['team_id', String(teamId)],
+        'and',
+        ['dataset_id', String(datasetId)],
+        'and',
+        ['collection_id', String(collectionId)]
+      ]
+    });
+
     return total;
   };
 }

packages/service/core/ai/config.ts

@@ -11,14 +11,17 @@ import { i18nT } from '../../../web/i18n/utils';
 import { OpenaiAccountType } from '@fastgpt/global/support/user/team/type';
 import { getLLMModel } from './model';
 
-export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
+const aiProxyBaseUrl = process.env.AIPROXY_API_ENDPOINT
+  ? `${process.env.AIPROXY_API_ENDPOINT}/v1`
+  : undefined;
+const openaiBaseUrl = aiProxyBaseUrl || process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
+const openaiBaseKey = process.env.AIPROXY_API_TOKEN || process.env.CHAT_API_KEY || '';
 
 export const getAIApi = (props?: { userKey?: OpenaiAccountType; timeout?: number }) => {
   const { userKey, timeout } = props || {};
 
   const baseUrl = userKey?.baseUrl || global?.systemEnv?.oneapiUrl || openaiBaseUrl;
-  const apiKey = userKey?.key || global?.systemEnv?.chatApiKey || process.env.CHAT_API_KEY || '';
-
+  const apiKey = userKey?.key || global?.systemEnv?.chatApiKey || openaiBaseKey;
   return new OpenAI({
     baseURL: baseUrl,
     apiKey,
@@ -32,7 +35,7 @@ export const getAxiosConfig = (props?: { userKey?: OpenaiAccountType }) => {
   const { userKey } = props || {};
 
   const baseUrl = userKey?.baseUrl || global?.systemEnv?.oneapiUrl || openaiBaseUrl;
-  const apiKey = userKey?.key || global?.systemEnv?.chatApiKey || process.env.CHAT_API_KEY || '';
+  const apiKey = userKey?.key || global?.systemEnv?.chatApiKey || openaiBaseKey;
 
   return {
     baseUrl,
@@ -72,6 +75,7 @@ export const createChatCompletion = async ({
       userKey,
       timeout: formatTimeout
     });
+
     const response = await ai.chat.completions.create(body, {
       ...options,
       ...(modelConstantsData.requestUrl ? { path: modelConstantsData.requestUrl } : {}),

packages/service/core/ai/config/provider/AliCloud.json

@@ -1,4 +1,10 @@
 {
   "provider": "AliCloud",
-  "list": []
-}
+  "list": [
+    {
+      "model": "SenseVoiceSmall",
+      "name": "SenseVoiceSmall",
+      "type": "stt"
+    }
+  ]
+}

packages/service/core/ai/config/provider/Claude.json

@@ -1,6 +1,30 @@
 {
   "provider": "Claude",
   "list": [
+    {
+      "model": "claude-3-7-sonnet-20250219",
+      "name": "claude-3-7-sonnet-20250219",
+      "maxContext": 200000,
+      "maxResponse": 8000,
+      "quoteMaxToken": 100000,
+      "maxTemperature": 1,
+      "showTopP": true,
+      "showStopSign": true,
+      "vision": true,
+      "toolChoice": true,
+      "functionCall": false,
+      "defaultSystemChatPrompt": "",
+      "datasetProcess": true,
+      "usedInClassify": true,
+      "customCQPrompt": "",
+      "usedInExtractFields": true,
+      "usedInQueryExtension": true,
+      "customExtractPrompt": "",
+      "usedInToolCall": true,
+      "defaultConfig": {},
+      "fieldMap": {},
+      "type": "llm"
+    },
     {
       "model": "claude-3-5-haiku-20241022",
       "name": "claude-3-5-haiku-20241022",
@@ -10,7 +34,7 @@
       "maxTemperature": 1,
       "showTopP": true,
       "showStopSign": true,
-      "vision": false,
+      "vision": true,
       "toolChoice": true,
       "functionCall": false,
       "defaultSystemChatPrompt": "",
@@ -98,4 +122,4 @@
       "type": "llm"
     }
   ]
-}
+}

packages/service/core/ai/config/provider/Gemini.json

@@ -1,6 +1,54 @@
 {
   "provider": "Gemini",
   "list": [
+    {
+      "model": "gemini-2.0-flash",
+      "name": "gemini-2.0-flash",
+      "maxContext": 1000000,
+      "maxResponse": 8000,
+      "quoteMaxToken": 60000,
+      "maxTemperature": 1,
+      "vision": true,
+      "toolChoice": true,
+      "functionCall": false,
+      "defaultSystemChatPrompt": "",
+      "datasetProcess": true,
+      "usedInClassify": true,
+      "customCQPrompt": "",
+      "usedInExtractFields": true,
+      "usedInQueryExtension": true,
+      "customExtractPrompt": "",
+      "usedInToolCall": true,
+      "defaultConfig": {},
+      "fieldMap": {},
+      "type": "llm",
+      "showTopP": true,
+      "showStopSign": true
+    },
+    {
+      "model": "gemini-2.0-pro-exp",
+      "name": "gemini-2.0-pro-exp",
+      "maxContext": 2000000,
+      "maxResponse": 8000,
+      "quoteMaxToken": 100000,
+      "maxTemperature": 1,
+      "vision": true,
+      "toolChoice": true,
+      "functionCall": false,
+      "defaultSystemChatPrompt": "",
+      "datasetProcess": true,
+      "usedInClassify": true,
+      "customCQPrompt": "",
+      "usedInExtractFields": true,
+      "usedInQueryExtension": true,
+      "customExtractPrompt": "",
+      "usedInToolCall": true,
+      "defaultConfig": {},
+      "fieldMap": {},
+      "type": "llm",
+      "showTopP": true,
+      "showStopSign": true
+    },
     {
       "model": "gemini-1.5-flash",
       "name": "gemini-1.5-flash",
@@ -153,4 +201,4 @@
       "type": "embedding"
     }
   ]
-}
+}

packages/service/core/ai/config/provider/Grok.json

@@ -1,4 +1,29 @@
 {
   "provider": "Grok",
-  "list": []
-}
+  "list": [
+    {
+      "model": "grok-3",
+      "name": "grok-3",
+      "maxContext": 128000,
+      "maxResponse": 8000,
+      "quoteMaxToken": 128000,
+      "maxTemperature": 1,
+      "showTopP": true,
+      "showStopSign": true,
+      "vision": false,
+      "toolChoice": false,
+      "functionCall": false,
+      "defaultSystemChatPrompt": "",
+      "datasetProcess": true,
+      "usedInClassify": true,
+      "customCQPrompt": "",
+      "usedInExtractFields": true,
+      "usedInQueryExtension": true,
+      "customExtractPrompt": "",
+      "usedInToolCall": true,
+      "defaultConfig": {},
+      "fieldMap": {},
+      "type": "llm"
+    }
+  ]
+}

packages/service/core/ai/config/utils.ts

@@ -52,6 +52,12 @@ export const loadSystemModels = async (init = false) => {
         if (model.isDefault) {
           global.systemDefaultModel.llm = model;
         }
+        if (model.isDefaultDatasetTextModel) {
+          global.systemDefaultModel.datasetTextLLM = model;
+        }
+        if (model.isDefaultDatasetImageModel) {
+          global.systemDefaultModel.datasetImageLLM = model;
+        }
       } else if (model.type === ModelTypeEnum.embedding) {
         global.embeddingModelMap.set(model.model, model);
         global.embeddingModelMap.set(model.name, model);
@@ -134,6 +140,16 @@ export const loadSystemModels = async (init = false) => {
     if (!global.systemDefaultModel.llm) {
       global.systemDefaultModel.llm = Array.from(global.llmModelMap.values())[0];
     }
+    if (!global.systemDefaultModel.datasetTextLLM) {
+      global.systemDefaultModel.datasetTextLLM = Array.from(global.llmModelMap.values()).find(
+        (item) => item.datasetProcess
+      );
+    }
+    if (!global.systemDefaultModel.datasetImageLLM) {
+      global.systemDefaultModel.datasetImageLLM = Array.from(global.llmModelMap.values()).find(
+        (item) => item.vision
+      );
+    }
     if (!global.systemDefaultModel.embedding) {
       global.systemDefaultModel.embedding = Array.from(global.embeddingModelMap.values())[0];
     }

packages/service/core/ai/type.d.ts

@@ -22,6 +22,9 @@ export type SystemModelItemType =
 
 export type SystemDefaultModelType = {
   [ModelTypeEnum.llm]?: LLMModelItemType;
+  datasetTextLLM?: LLMModelItemType;
+  datasetImageLLM?: LLMModelItemType;
+
   [ModelTypeEnum.embedding]?: EmbeddingModelItemType;
   [ModelTypeEnum.tts]?: TTSModelType;
   [ModelTypeEnum.stt]?: STTModelType;

packages/service/core/ai/utils.ts

@@ -95,11 +95,145 @@ export const llmCompletionsBodyFormat = <T extends CompletionsBodyType>(
   return requestBody as unknown as InferCompletionsBody<T>;
 };
 
-export const llmStreamResponseToText = async (response: StreamChatType) => {
+export const llmStreamResponseToAnswerText = async (response: StreamChatType) => {
   let answer = '';
   for await (const part of response) {
     const content = part.choices?.[0]?.delta?.content || '';
     answer += content;
   }
-  return answer;
+  return parseReasoningContent(answer)[1];
+};
+
+// Parse <think></think> tags to think and answer - unstream response
+export const parseReasoningContent = (text: string): [string, string] => {
+  const regex = /<think>([\s\S]*?)<\/think>/;
+  const match = text.match(regex);
+
+  if (!match) {
+    return ['', text];
+  }
+
+  const thinkContent = match[1].trim();
+
+  // Add answer (remaining text after think tag)
+  const answerContent = text.slice(match.index! + match[0].length);
+
+  return [thinkContent, answerContent];
+};
+
+// Parse <think></think> tags to think and answer - stream response
+export const parseReasoningStreamContent = () => {
+  let isInThinkTag: boolean | undefined;
+
+  const startTag = '<think>';
+  let startTagBuffer = '';
+
+  const endTag = '</think>';
+  let endTagBuffer = '';
+
+  /* 
+    parseReasoning - 只控制是否主动解析 <think></think>，如果接口已经解析了，仍然会返回 think 内容。
+  */
+  const parsePart = (
+    part: {
+      choices: {
+        delta: {
+          content?: string;
+          reasoning_content?: string;
+        };
+      }[];
+    },
+    parseReasoning = false
+  ): [string, string] => {
+    const content = part.choices?.[0]?.delta?.content || '';
+
+    // @ts-ignore
+    const reasoningContent = part.choices?.[0]?.delta?.reasoning_content || '';
+    if (reasoningContent || !parseReasoning) {
+      isInThinkTag = false;
+      return [reasoningContent, content];
+    }
+
+    if (!content) {
+      return ['', ''];
+    }
+
+    // 如果不在 think 标签中，或者有 reasoningContent(接口已解析），则返回 reasoningContent 和 content
+    if (isInThinkTag === false) {
+      return ['', content];
+    }
+
+    // 检测是否为 think 标签开头的数据
+    if (isInThinkTag === undefined) {
+      // Parse content think and answer
+      startTagBuffer += content;
+      // 太少内容时候，暂时不解析
+      if (startTagBuffer.length < startTag.length) {
+        return ['', ''];
+      }
+
+      if (startTagBuffer.startsWith(startTag)) {
+        isInThinkTag = true;
+        return [startTagBuffer.slice(startTag.length), ''];
+      }
+
+      // 如果未命中 think 标签，则认为不在 think 标签中，返回 buffer 内容作为 content
+      isInThinkTag = false;
+      return ['', startTagBuffer];
+    }
+
+    // 确认是 think 标签内容，开始返回 think 内容，并实时检测 </think>
+    /* 
+      检测 </think> 方案。
+      存储所有疑似 </think> 的内容，直到检测到完整的 </think> 标签或超出 </think> 长度。
+      content 返回值包含以下几种情况:
+        abc - 完全未命中尾标签
+        abc<th - 命中一部分尾标签
+        abc</think> - 完全命中尾标签
+        abc</think>abc - 完全命中尾标签
+        </think>abc - 完全命中尾标签
+        k>abc - 命中一部分尾标签
+    */
+    // endTagBuffer 专门用来记录疑似尾标签的内容
+    if (endTagBuffer) {
+      endTagBuffer += content;
+      if (endTagBuffer.includes(endTag)) {
+        isInThinkTag = false;
+        const answer = endTagBuffer.slice(endTag.length);
+        return ['', answer];
+      } else if (endTagBuffer.length >= endTag.length) {
+        // 缓存内容超出尾标签长度，且仍未命中 </think>，则认为本次猜测 </think> 失败，仍处于 think 阶段。
+        const tmp = endTagBuffer;
+        endTagBuffer = '';
+        return [tmp, ''];
+      }
+      return ['', ''];
+    } else if (content.includes(endTag)) {
+      // 返回内容，完整命中</think>，直接结束
+      isInThinkTag = false;
+      const [think, answer] = content.split(endTag);
+      return [think, answer];
+    } else {
+      // 无 buffer，且未命中 </think>，开始疑似 </think> 检测。
+      for (let i = 1; i < endTag.length; i++) {
+        const partialEndTag = endTag.slice(0, i);
+        // 命中一部分尾标签
+        if (content.endsWith(partialEndTag)) {
+          const think = content.slice(0, -partialEndTag.length);
+          endTagBuffer += partialEndTag;
+          return [think, ''];
+        }
+      }
+    }
+
+    // 完全未命中尾标签，还是 think 阶段。
+    return [content, ''];
+  };
+
+  const getStartTagBuffer = () => startTagBuffer;
+
+  return {
+    parsePart,
+    getStartTagBuffer
+  };
 };

packages/service/core/chat/chatSchema.ts

@@ -1,7 +1,7 @@
 import { connectionMongo, getMongoModel } from '../../common/mongo';
 const { Schema } = connectionMongo;
 import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d';
-import { ChatSourceMap } from '@fastgpt/global/core/chat/constants';
+import { ChatSourceEnum, ChatSourceMap } from '@fastgpt/global/core/chat/constants';
 import {
   TeamCollectionName,
   TeamMemberCollectionName
@@ -52,8 +52,10 @@ const ChatSchema = new Schema({
   },
   source: {
     type: String,
-    required: true
+    required: true,
+    enum: Object.values(ChatSourceEnum)
   },
+  sourceName: String,
   shareId: {
     type: String
   },

packages/service/core/chat/saveChat.ts

@@ -1,6 +1,10 @@
 import type { AIChatItemType, UserChatItemType } from '@fastgpt/global/core/chat/type.d';
 import { MongoApp } from '../app/schema';
-import { ChatItemValueTypeEnum, ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
+import {
+  ChatItemValueTypeEnum,
+  ChatRoleEnum,
+  ChatSourceEnum
+} from '@fastgpt/global/core/chat/constants';
 import { MongoChatItem } from './chatItemSchema';
 import { MongoChat } from './chatSchema';
 import { addLog } from '../../common/system/log';
@@ -22,7 +26,8 @@ type Props = {
   variables?: Record<string, any>;
   isUpdateUseTime: boolean;
   newTitle: string;
-  source: string;
+  source: `${ChatSourceEnum}`;
+  sourceName?: string;
   shareId?: string;
   outLinkUid?: string;
   content: [UserChatItemType & { dataId?: string }, AIChatItemType & { dataId?: string }];
@@ -40,6 +45,7 @@ export async function saveChat({
   isUpdateUseTime,
   newTitle,
   source,
+  sourceName,
   shareId,
   outLinkUid,
   content,
@@ -96,6 +102,7 @@ export async function saveChat({
             pluginInputs,
             title: newTitle,
             source,
+            sourceName,
             shareId,
             outLinkUid,
             metadata: metadataUpdate,

packages/service/core/dataset/collection/controller.ts

@@ -25,6 +25,7 @@ import { MongoImage } from '../../../common/file/image/schema';
 import { hashStr } from '@fastgpt/global/common/string/tools';
 import { addDays } from 'date-fns';
 import { MongoDatasetDataText } from '../data/dataTextSchema';
+import { delay, retryFn } from '@fastgpt/global/common/system/utils';
 
 export const createCollectionAndInsertData = async ({
   dataset,
@@ -216,7 +217,7 @@ export async function createOneCollection({
         nextSyncTime
       }
     ],
-    { session }
+    { session, ordered: true }
   );
 
   return collection;
@@ -227,8 +228,14 @@ export const delCollectionRelatedSource = async ({
   collections,
   session
 }: {
-  collections: DatasetCollectionSchemaType[];
-  session: ClientSession;
+  collections: {
+    teamId: string;
+    fileId?: string;
+    metadata?: {
+      relatedImgId?: string;
+    };
+  }[];
+  session?: ClientSession;
 }) => {
   if (collections.length === 0) return;
 
@@ -259,11 +266,13 @@ export const delCollectionRelatedSource = async ({
 export async function delCollection({
   collections,
   session,
-  delRelatedSource
+  delImg = true,
+  delFile = true
 }: {
   collections: DatasetCollectionSchemaType[];
   session: ClientSession;
-  delRelatedSource: boolean;
+  delImg: boolean;
+  delFile: boolean;
 }) {
   if (collections.length === 0) return;
 
@@ -274,83 +283,55 @@ export async function delCollection({
   const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId))));
   const collectionIds = collections.map((item) => String(item._id));
 
-  // Delete training data
-  await MongoDatasetTraining.deleteMany({
-    teamId,
-    datasetIds: { $in: datasetIds },
-    collectionId: { $in: collectionIds }
+  await retryFn(async () => {
+    await Promise.all([
+      // Delete training data
+      MongoDatasetTraining.deleteMany({
+        teamId,
+        datasetId: { $in: datasetIds },
+        collectionId: { $in: collectionIds }
+      }),
+      // Delete dataset_data_texts
+      MongoDatasetDataText.deleteMany({
+        teamId,
+        datasetId: { $in: datasetIds },
+        collectionId: { $in: collectionIds }
+      }),
+      // Delete dataset_datas
+      MongoDatasetData.deleteMany({
+        teamId,
+        datasetId: { $in: datasetIds },
+        collectionId: { $in: collectionIds }
+      }),
+      ...(delImg
+        ? [
+            delImgByRelatedId({
+              teamId,
+              relateIds: collections
+                .map((item) => item?.metadata?.relatedImgId || '')
+                .filter(Boolean)
+            })
+          ]
+        : []),
+      ...(delFile
+        ? [
+            delFileByFileIdList({
+              bucketName: BucketNameEnum.dataset,
+              fileIdList: collections.map((item) => item?.fileId || '').filter(Boolean)
+            })
+          ]
+        : []),
+      // Delete vector data
+      deleteDatasetDataVector({ teamId, datasetIds, collectionIds })
+    ]);
+
+    // delete collections
+    await MongoDatasetCollection.deleteMany(
+      {
+        teamId,
+        _id: { $in: collectionIds }
+      },
+      { session }
+    );
   });
-
-  /* file and imgs */
-  if (delRelatedSource) {
-    await delCollectionRelatedSource({ collections, session });
-  }
-
-  // Delete dataset_datas
-  await MongoDatasetData.deleteMany(
-    { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
-    { session }
-  );
-  // Delete dataset_data_texts
-  await MongoDatasetDataText.deleteMany(
-    { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
-    { session }
-  );
-
-  // delete collections
-  await MongoDatasetCollection.deleteMany(
-    {
-      teamId,
-      _id: { $in: collectionIds }
-    },
-    { session }
-  );
-
-  // no session delete: delete files, vector data
-  await deleteDatasetDataVector({ teamId, datasetIds, collectionIds });
-}
-
-/**
- * delete delOnlyCollection
- */
-export async function delOnlyCollection({
-  collections,
-  session
-}: {
-  collections: DatasetCollectionSchemaType[];
-  session: ClientSession;
-}) {
-  if (collections.length === 0) return;
-
-  const teamId = collections[0].teamId;
-
-  if (!teamId) return Promise.reject('teamId is not exist');
-
-  const datasetIds = Array.from(new Set(collections.map((item) => String(item.datasetId))));
-  const collectionIds = collections.map((item) => String(item._id));
-
-  // delete training data
-  await MongoDatasetTraining.deleteMany({
-    teamId,
-    datasetIds: { $in: datasetIds },
-    collectionId: { $in: collectionIds }
-  });
-
-  // delete dataset.datas
-  await MongoDatasetData.deleteMany(
-    { teamId, datasetIds: { $in: datasetIds }, collectionId: { $in: collectionIds } },
-    { session }
-  );
-
-  // delete collections
-  await MongoDatasetCollection.deleteMany(
-    {
-      teamId,
-      _id: { $in: collectionIds }
-    },
-    { session }
-  );
-
-  // no session delete: delete files, vector data
-  await deleteDatasetDataVector({ teamId, datasetIds, collectionIds });
 }

packages/service/core/dataset/collection/utils.ts

@@ -97,7 +97,7 @@ export const createOrGetCollectionTags = async ({
       datasetId,
       tag: tagContent
     })),
-    { session }
+    { session, ordered: true }
   );
 
   return [...existingTags.map((tag) => tag._id), ...newTags.map((tag) => tag._id)];
@@ -174,6 +174,14 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
   }
 
   await mongoSessionRun(async (session) => {
+    // Delete old collection
+    await delCollection({
+      collections: [collection],
+      delImg: false,
+      delFile: false,
+      session
+    });
+
     // Create new collection
     await createCollectionAndInsertData({
       session,
@@ -208,13 +216,6 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
         updateTime: new Date()
       }
     });
-
-    // Delete old collection
-    await delCollection({
-      collections: [collection],
-      delRelatedSource: false,
-      session
-    });
   });
 
   return DatasetCollectionSyncResultEnum.success;

packages/service/core/dataset/controller.ts

@@ -7,6 +7,8 @@ import { MongoDatasetTraining } from './training/schema';
 import { MongoDatasetData } from './data/schema';
 import { deleteDatasetDataVector } from '../../common/vectorStore/controller';
 import { MongoDatasetDataText } from './data/dataTextSchema';
+import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
+import { retryFn } from '@fastgpt/global/common/system/utils';
 
 /* ============= dataset ========== */
 /* find all datasetId by top datasetId */
@@ -54,7 +56,7 @@ export async function getCollectionWithDataset(collectionId: string) {
     .populate<{ dataset: DatasetSchemaType }>('dataset')
     .lean();
   if (!data) {
-    return Promise.reject('Collection is not exist');
+    return Promise.reject(DatasetErrEnum.unExistCollection);
   }
   return data;
 }
@@ -77,40 +79,39 @@ export async function delDatasetRelevantData({
 
   const datasetIds = datasets.map((item) => item._id);
 
-  // delete training data
-  await MongoDatasetTraining.deleteMany({
-    teamId,
-    datasetId: { $in: datasetIds }
-  });
-
   // Get _id, teamId, fileId, metadata.relatedImgId for all collections
   const collections = await MongoDatasetCollection.find(
     {
       teamId,
       datasetId: { $in: datasetIds }
     },
-    '_id teamId datasetId fileId metadata',
-    { session }
+    '_id teamId datasetId fileId metadata'
   ).lean();
 
-  // Delete Image and file
-  await delCollectionRelatedSource({ collections, session });
+  await retryFn(async () => {
+    await Promise.all([
+      // delete training data
+      MongoDatasetTraining.deleteMany({
+        teamId,
+        datasetId: { $in: datasetIds }
+      }),
+      //Delete dataset_data_texts
+      MongoDatasetDataText.deleteMany({
+        teamId,
+        datasetId: { $in: datasetIds }
+      }),
+      //delete dataset_datas
+      MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } }),
+      // Delete Image and file
+      delCollectionRelatedSource({ collections }),
+      // Delete vector data
+      deleteDatasetDataVector({ teamId, datasetIds })
+    ]);
+  });
 
   // delete collections
   await MongoDatasetCollection.deleteMany({
     teamId,
     datasetId: { $in: datasetIds }
   }).session(session);
-
-  // No session delete:
-  // Delete dataset_data_texts
-  await MongoDatasetDataText.deleteMany({
-    teamId,
-    datasetId: { $in: datasetIds }
-  });
-  // delete dataset_datas
-  await MongoDatasetData.deleteMany({ teamId, datasetId: { $in: datasetIds } });
-
-  // Delete vector data
-  await deleteDatasetDataVector({ teamId, datasetIds });
 }

packages/service/core/dataset/data/dataTextSchema.ts

@@ -1,6 +1,6 @@
 import { connectionMongo, getMongoModel } from '../../../common/mongo';
 const { Schema } = connectionMongo;
-import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
+import { DatasetDataTextSchemaType } from '@fastgpt/global/core/dataset/type.d';
 import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
 import { DatasetCollectionName } from '../schema';
 import { DatasetColCollectionName } from '../collection/schema';
@@ -40,12 +40,13 @@ try {
       default_language: 'none'
     }
   );
+  DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, collectionId: 1 });
   DatasetDataTextSchema.index({ dataId: 1 }, { unique: true });
 } catch (error) {
   console.log(error);
 }
 
-export const MongoDatasetDataText = getMongoModel<DatasetDataSchemaType>(
+export const MongoDatasetDataText = getMongoModel<DatasetDataTextSchemaType>(
   DatasetDataTextCollectionName,
   DatasetDataTextSchema
 );

packages/service/core/dataset/search/controller.ts

@@ -200,6 +200,7 @@ export async function searchDatasetData(
       forbidCollectionIdList: collections.map((item) => String(item._id))
     };
   };
+
   /* 
     Collection metadata filter
     标签过滤：
@@ -207,6 +208,63 @@ export async function searchDatasetData(
     2. and 标签和 null 不能共存，否则返回空数组
   */
   const filterCollectionByMetadata = async (): Promise<string[] | undefined> => {
+    const getAllCollectionIds = async ({
+      parentCollectionIds
+    }: {
+      parentCollectionIds?: string[];
+    }): Promise<string[] | undefined> => {
+      if (!parentCollectionIds) return;
+      if (parentCollectionIds.length === 0) {
+        return [];
+      }
+
+      const collections = await MongoDatasetCollection.find(
+        {
+          teamId,
+          datasetId: { $in: datasetIds },
+          _id: { $in: parentCollectionIds }
+        },
+        '_id type',
+        {
+          ...readFromSecondary
+        }
+      ).lean();
+
+      const resultIds = new Set<string>();
+      collections.forEach((item) => {
+        if (item.type !== 'folder') {
+          resultIds.add(String(item._id));
+        }
+      });
+
+      const folderIds = collections
+        .filter((item) => item.type === 'folder')
+        .map((item) => String(item._id));
+
+      // Get all child collection ids
+      if (folderIds.length) {
+        const childCollections = await MongoDatasetCollection.find(
+          {
+            teamId,
+            datasetId: { $in: datasetIds },
+            parentId: { $in: folderIds }
+          },
+          '_id type',
+          {
+            ...readFromSecondary
+          }
+        ).lean();
+
+        const childIds = await getAllCollectionIds({
+          parentCollectionIds: childCollections.map((item) => String(item._id))
+        });
+
+        childIds?.forEach((id) => resultIds.add(id));
+      }
+
+      return Array.from(resultIds);
+    };
+
     if (!collectionFilterMatch || !global.feConfigs.isPlus) return;
 
     let tagCollectionIdList: string[] | undefined = undefined;
@@ -326,13 +384,19 @@ export async function searchDatasetData(
       }
 
       // Concat tag and time
-      if (tagCollectionIdList && createTimeCollectionIdList) {
-        return tagCollectionIdList.filter((id) => createTimeCollectionIdList!.includes(id));
-      } else if (tagCollectionIdList) {
-        return tagCollectionIdList;
-      } else if (createTimeCollectionIdList) {
-        return createTimeCollectionIdList;
-      }
+      const collectionIds = (() => {
+        if (tagCollectionIdList && createTimeCollectionIdList) {
+          return tagCollectionIdList.filter((id) =>
+            (createTimeCollectionIdList as string[]).includes(id)
+          );
+        }
+
+        return tagCollectionIdList || createTimeCollectionIdList;
+      })();
+
+      return await getAllCollectionIds({
+        parentCollectionIds: collectionIds
+      });
     } catch (error) {}
   };
   const embeddingRecall = async ({
@@ -383,6 +447,7 @@ export async function searchDatasetData(
       ).lean()
     ]);
 
+    const set = new Map<string, number>();
     const formatResult = results
       .map((item, index) => {
         const collection = collections.find((col) => String(col._id) === String(item.collectionId));
@@ -398,8 +463,6 @@ export async function searchDatasetData(
           return;
         }
 
-        const score = item?.score || 0;
-
         const result: SearchDataResponseItemType = {
           id: String(data._id),
           updateTime: data.updateTime,
@@ -409,12 +472,24 @@ export async function searchDatasetData(
           datasetId: String(data.datasetId),
           collectionId: String(data.collectionId),
           ...getCollectionSourceData(collection),
-          score: [{ type: SearchScoreTypeEnum.embedding, value: score, index }]
+          score: [{ type: SearchScoreTypeEnum.embedding, value: item?.score || 0, index }]
         };
 
         return result;
       })
-      .filter(Boolean) as SearchDataResponseItemType[];
+      .filter((item) => {
+        if (!item) return false;
+        if (set.has(item.id)) return false;
+        set.set(item.id, 1);
+        return true;
+      })
+      .map((item, index) => {
+        if (!item) return;
+        return {
+          ...item,
+          score: item.score.map((item) => ({ ...item, index }))
+        };
+      }) as SearchDataResponseItemType[];
 
     return {
       embeddingRecallResults: formatResult,
@@ -717,11 +792,12 @@ export const defaultSearchDatasetData = async ({
     ? getLLMModel(datasetSearchExtensionModel)
     : undefined;
 
-  const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
-    query,
-    extensionModel,
-    extensionBg: datasetSearchExtensionBg
-  });
+  const { concatQueries, extensionQueries, rewriteQuery, aiExtensionResult } =
+    await datasetSearchQueryExtension({
+      query,
+      extensionModel,
+      extensionBg: datasetSearchExtensionBg
+    });
 
   const result = await searchDatasetData({
     ...props,
@@ -736,7 +812,7 @@ export const defaultSearchDatasetData = async ({
           model: aiExtensionResult.model,
           inputTokens: aiExtensionResult.inputTokens,
           outputTokens: aiExtensionResult.outputTokens,
-          query: concatQueries.join('\n')
+          query: extensionQueries.join('\n')
         }
       : undefined
   };

packages/service/core/dataset/search/utils.ts

@@ -72,12 +72,15 @@ Human: ${query}
     if (result.extensionQueries?.length === 0) return;
     return result;
   })();
+
+  const extensionQueries = filterSamQuery(aiExtensionResult?.extensionQueries || []);
   if (aiExtensionResult) {
-    queries = filterSamQuery(queries.concat(aiExtensionResult.extensionQueries));
+    queries = filterSamQuery(queries.concat(extensionQueries));
     rewriteQuery = queries.join('\n');
   }
 
   return {
+    extensionQueries,
     concatQueries: queries,
     rewriteQuery,
     aiExtensionResult

packages/service/core/workflow/dispatch/chat/oneapi.ts

@@ -3,11 +3,8 @@ import { filterGPTMessageByMaxContext, loadRequestMessages } from '../../../chat
 import type { ChatItemType, UserChatItemValueItemType } from '@fastgpt/global/core/chat/type.d';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
-import {
-  parseReasoningContent,
-  parseReasoningStreamContent,
-  textAdaptGptResponse
-} from '@fastgpt/global/core/workflow/runtime/utils';
+import { textAdaptGptResponse } from '@fastgpt/global/core/workflow/runtime/utils';
+import { parseReasoningContent, parseReasoningStreamContent } from '../../../ai/utils';
 import { createChatCompletion } from '../../../ai/config';
 import type { ChatCompletionMessageParam, StreamChatType } from '@fastgpt/global/core/ai/type.d';
 import { formatModelChars2Points } from '../../../../support/wallet/usage/utils';

packages/service/package.json

@@ -24,7 +24,7 @@
     "jsonwebtoken": "^9.0.2",
     "lodash": "^4.17.21",
     "mammoth": "^1.6.0",
-    "mongoose": "^7.0.2",
+    "mongoose": "^8.10.1",
     "multer": "1.4.5-lts.1",
     "next": "14.2.5",
     "nextjs-cors": "^2.2.0",

packages/service/support/permission/controller.ts

@@ -178,7 +178,7 @@ export const getClbsAndGroupsWithInfo = async ({
   ]);
 
 export const delResourcePermissionById = (id: string) => {
-  return MongoResourcePermission.findByIdAndRemove(id);
+  return MongoResourcePermission.findByIdAndDelete(id);
 };
 export const delResourcePermission = ({
   session,

packages/service/support/permission/inheritPermission.ts

@@ -196,7 +196,8 @@ export async function syncCollaborators({
       permission: item.permission
     })),
     {
-      session
+      session,
+      ordered: true
     }
   );
 }

packages/service/support/permission/publish/authLink.ts

@@ -54,7 +54,7 @@ export async function authOutLinkCrud({
 }
 
 /* outLink exist and it app exist */
-export async function authOutLinkValid<T extends OutlinkAppType = undefined>({
+export async function authOutLinkValid<T extends OutlinkAppType = any>({
   shareId
 }: {
   shareId?: string;
@@ -62,7 +62,7 @@ export async function authOutLinkValid<T extends OutlinkAppType = undefined>({
   if (!shareId) {
     return Promise.reject(OutLinkErrEnum.linkUnInvalid);
   }
-  const outLinkConfig = (await MongoOutLink.findOne({ shareId }).lean()) as OutLinkSchema<T>;
+  const outLinkConfig = await MongoOutLink.findOne({ shareId }).lean<OutLinkSchema<T>>();
 
   if (!outLinkConfig) {
     return Promise.reject(OutLinkErrEnum.linkUnInvalid);
@@ -70,6 +70,6 @@ export async function authOutLinkValid<T extends OutlinkAppType = undefined>({
 
   return {
     appId: outLinkConfig.appId,
-    outLinkConfig
+    outLinkConfig: outLinkConfig
   };
 }

packages/service/support/permission/teamLimit.ts

@@ -64,7 +64,7 @@ export const checkTeamDatasetLimit = async (teamId: string) => {
 export const checkTeamAppLimit = async (teamId: string, amount = 1) => {
   const [{ standardConstants }, appCount] = await Promise.all([
     getTeamStandPlan({ teamId }),
-    MongoApp.count({
+    MongoApp.countDocuments({
       teamId,
       type: { $in: [AppTypeEnum.simple, AppTypeEnum.workflow, AppTypeEnum.plugin] }
     })

packages/service/support/user/controller.ts

@@ -46,6 +46,7 @@ export async function getUserDetail({
     promotionRate: user.promotionRate,
     team: tmb,
     notificationAccount: tmb.notificationAccount,
-    permission: tmb.permission
+    permission: tmb.permission,
+    contact: user.contact
   };
 }

packages/service/support/user/schema.ts

@@ -57,6 +57,7 @@ const UserSchema = new Schema({
   },
   fastgpt_sem: Object,
   sourceDomain: String,
+  contact: String,
 
   /** @deprecated */
   avatar: String

packages/service/support/user/team/controller.ts

@@ -15,7 +15,7 @@ import { TeamDefaultPermissionVal } from '@fastgpt/global/support/permission/use
 import { MongoMemberGroupModel } from '../../permission/memberGroup/memberGroupSchema';
 import { mongoSessionRun } from '../../../common/mongo/sessionRun';
 import { DefaultGroupName } from '@fastgpt/global/support/user/team/group/constant';
-import { getAIApi, openaiBaseUrl } from '../../../core/ai/config';
+import { getAIApi } from '../../../core/ai/config';
 import { createRootOrg } from '../../permission/org/controllers';
 import { refreshSourceAvatar } from '../../../common/file/image/controller';
 
@@ -152,7 +152,7 @@ export async function updateTeam({
   // auth openai key
   if (openaiAccount?.key) {
     console.log('auth user openai key', openaiAccount?.key);
-    const baseUrl = openaiAccount?.baseUrl || openaiBaseUrl;
+    const baseUrl = openaiAccount?.baseUrl || 'https://api.openai.com/v1';
     openaiAccount.baseUrl = baseUrl;
 
     const ai = getAIApi({

packages/service/support/user/team/teamMemberSchema.ts

@@ -36,6 +36,9 @@ const TeamMemberSchema = new Schema({
     type: Date,
     default: () => new Date()
   },
+  updateTime: {
+    type: Date
+  },
   defaultTeam: {
     type: Boolean,
     default: false

packages/service/support/wallet/sub/utils.ts

@@ -100,7 +100,7 @@ export const initTeamFreePlan = async ({
         surplusPoints: freePoints
       }
     ],
-    { session }
+    { session, ordered: true }
   );
 };
 

packages/service/support/wallet/usage/controller.ts

@@ -8,12 +8,12 @@ import { i18nT } from '../../../../web/i18n/utils';
 import { pushConcatBillTask, pushReduceTeamAiPointsTask } from './utils';
 
 import { POST } from '../../../common/api/plusRequest';
-import { FastGPTProUrl } from '../../../common/system/constants';
+import { isFastGPTMainService } from '../../../common/system/constants';
 
 export async function createUsage(data: CreateUsageProps) {
   try {
     // In FastGPT server
-    if (FastGPTProUrl) {
+    if (isFastGPTMainService) {
       await POST('/support/wallet/usage/createUsage', data);
     } else if (global.reduceAiPointsQueue) {
       // In FastGPT pro server
@@ -31,7 +31,7 @@ export async function createUsage(data: CreateUsageProps) {
 export async function concatUsage(data: ConcatUsageProps) {
   try {
     // In FastGPT server
-    if (FastGPTProUrl) {
+    if (isFastGPTMainService) {
       await POST('/support/wallet/usage/concatUsage', data);
     } else if (global.reduceAiPointsQueue) {
       const {
@@ -160,7 +160,7 @@ export const createTrainingUsage = async ({
         ]
       }
     ],
-    { session }
+    { session, ordered: true }
   );
 
   return { billId: String(_id) };

packages/service/worker/readFile/parseOffice.ts

@@ -45,11 +45,11 @@ const parsePowerPoint = async ({
 
   // Returning an array of all the xml contents read using fs.readFileSync
   const xmlContentArray = await Promise.all(
-    files.map((file) => {
+    files.map(async (file) => {
       try {
-        return fs.promises.readFile(`${decompressPath}/${file.path}`, encoding);
+        return await fs.promises.readFile(`${decompressPath}/${file.path}`, encoding);
       } catch (err) {
-        return fs.promises.readFile(`${decompressPath}/${file.path}`, 'utf-8');
+        return await fs.promises.readFile(`${decompressPath}/${file.path}`, 'utf-8');
       }
     })
   );

packages/web/components/common/DateRangePicker/index.tsx

@@ -100,6 +100,13 @@ const DateRangePicker = ({
               if (date?.to === undefined) {
                 date.to = date.from;
               }
+
+              if (date?.from) {
+                date.from = new Date(date.from.setHours(0, 0, 0, 0));
+              }
+              if (date?.to) {
+                date.to = new Date(date.to.setHours(23, 59, 59, 999));
+              }
               setRange(date);
               onChange?.(date);
             }}

packages/web/components/common/Icon/constants.ts

@@ -6,6 +6,7 @@ export const iconPaths = {
   chatSend: () => import('./icons/chatSend.svg'),
   check: () => import('./icons/check.svg'),
   checkCircle: () => import('./icons/checkCircle.svg'),
+  close: () => import('./icons/close.svg'),
   closeSolid: () => import('./icons/closeSolid.svg'),
   code: () => import('./icons/code.svg'),
   collectionLight: () => import('./icons/collectionLight.svg'),
@@ -26,13 +27,16 @@ export const iconPaths = {
   'common/closeLight': () => import('./icons/common/closeLight.svg'),
   'common/confirm/commonTip': () => import('./icons/common/confirm/commonTip.svg'),
   'common/confirm/deleteTip': () => import('./icons/common/confirm/deleteTip.svg'),
+  'common/confirm/restoreTip': () => import('./icons/common/confirm/restoreTip.svg'),
   'common/confirm/rightTip': () => import('./icons/common/confirm/rightTip.svg'),
   'common/courseLight': () => import('./icons/common/courseLight.svg'),
   'common/customTitleLight': () => import('./icons/common/customTitleLight.svg'),
   'common/data': () => import('./icons/common/data.svg'),
   'common/dingtalkFill': () => import('./icons/common/dingtalkFill.svg'),
+  'common/disable': () => import('./icons/common/disable.svg'),
   'common/downArrowFill': () => import('./icons/common/downArrowFill.svg'),
   'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'),
+  'common/enable': () => import('./icons/common/enable.svg'),
   'common/errorFill': () => import('./icons/common/errorFill.svg'),
   'common/file/move': () => import('./icons/common/file/move.svg'),
   'common/folderFill': () => import('./icons/common/folderFill.svg'),
@@ -160,6 +164,7 @@ export const iconPaths = {
   'core/chat/chatLight': () => import('./icons/core/chat/chatLight.svg'),
   'core/chat/chatModelTag': () => import('./icons/core/chat/chatModelTag.svg'),
   'core/chat/chevronDown': () => import('./icons/core/chat/chevronDown.svg'),
+  'core/chat/chevronLeft': () => import('./icons/core/chat/chevronLeft.svg'),
   'core/chat/chevronRight': () => import('./icons/core/chat/chevronRight.svg'),
   'core/chat/chevronSelector': () => import('./icons/core/chat/chevronSelector.svg'),
   'core/chat/chevronUp': () => import('./icons/core/chat/chevronUp.svg'),
@@ -328,6 +333,7 @@ export const iconPaths = {
   edit: () => import('./icons/edit.svg'),
   empty: () => import('./icons/empty.svg'),
   export: () => import('./icons/export.svg'),
+  feedback: () => import('./icons/feedback.svg'),
   'file/csv': () => import('./icons/file/csv.svg'),
   'file/fill/csv': () => import('./icons/file/fill/csv.svg'),
   'file/fill/doc': () => import('./icons/file/fill/doc.svg'),
@@ -387,9 +393,9 @@ export const iconPaths = {
   'model/moonshot': () => import('./icons/model/moonshot.svg'),
   'model/ollama': () => import('./icons/model/ollama.svg'),
   'model/openai': () => import('./icons/model/openai.svg'),
+  'model/ppio': () => import('./icons/model/ppio.svg'),
   'model/qwen': () => import('./icons/model/qwen.svg'),
   'model/siliconflow': () => import('./icons/model/siliconflow.svg'),
-  'model/ppio': () => import('./icons/model/ppio.svg'),
   'model/sparkDesk': () => import('./icons/model/sparkDesk.svg'),
   'model/stepfun': () => import('./icons/model/stepfun.svg'),
   'model/yi': () => import('./icons/model/yi.svg'),

packages/web/components/common/Icon/icons/close.svg

@@ -0,0 +1,3 @@
+<svg  viewBox="0 0 13 12" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M9.65613 2.84383C9.85139 3.0391 9.85139 3.35568 9.65613 3.55094L7.20708 5.99999L9.65617 8.44908C9.85143 8.64434 9.85143 8.96093 9.65617 9.15619C9.46091 9.35145 9.14433 9.35145 8.94906 9.15619L6.49997 6.7071L4.05088 9.15619C3.85562 9.35145 3.53904 9.35145 3.34377 9.15619C3.14851 8.96093 3.14851 8.64434 3.34377 8.44908L5.79286 5.99999L3.34382 3.55094C3.14855 3.35568 3.14855 3.0391 3.34382 2.84383C3.53908 2.64857 3.85566 2.64857 4.05092 2.84383L6.49997 5.29288L8.94902 2.84383C9.14428 2.64857 9.46087 2.64857 9.65613 2.84383Z" fill="#92A5C9"/>
+</svg>

packages/web/components/common/Icon/icons/common/confirm/deleteTip.svg

@@ -1 +1,3 @@
-<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701403554068" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5098" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M572.074667 337.134933a57.275733 57.275733 0 1 0-114.176-6.007466h-0.170667l12.936533 272.896v0.443733a44.544 44.544 0 1 0 89.019734-1.194667l12.424533-266.1376z m-196.949334-191.214933c76.151467-130.048 199.816533-129.774933 275.797334 0l340.3776 581.085867c76.117333 130.048 15.7696 235.4176-135.236267 235.4176H169.984c-150.8352 0-211.217067-105.6768-135.202133-235.4176L375.125333 145.92z m140.049067 687.581867a57.275733 57.275733 0 1 0 0-114.517334 57.275733 57.275733 0 0 0 0 114.517334z" fill="#FB6547" p-id="5099"></path></svg>
+<svg  viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M8.77324 1.94537C9.14764 1.73458 9.57006 1.62384 9.99973 1.62384C10.4294 1.62384 10.8518 1.73458 11.2262 1.94537C11.6006 2.15617 11.9144 2.4599 12.1372 2.82727L12.1396 2.83123L19.198 14.6146L19.2047 14.6261C19.423 15.0041 19.5385 15.4327 19.5397 15.8692C19.541 16.3058 19.4279 16.735 19.2117 17.1142C18.9955 17.4935 18.6838 17.8095 18.3076 18.0309C17.9314 18.2523 17.5037 18.3713 17.0672 18.3761L17.0581 18.3762L2.93224 18.3761C2.49574 18.3713 2.0681 18.2523 1.69188 18.0309C1.31565 17.8095 1.00394 17.4935 0.787774 17.1142C0.571604 16.735 0.458504 16.3058 0.459727 15.8692C0.460949 15.4327 0.576451 15.0041 0.79474 14.6261L0.80151 14.6146L7.86222 2.82726C8.08506 2.4599 8.39883 2.15617 8.77324 1.94537ZM9.99973 3.29051C9.85651 3.29051 9.7157 3.32742 9.5909 3.39769C9.46666 3.46763 9.36246 3.56828 9.28824 3.68999L2.23531 15.4643C2.16432 15.5891 2.12679 15.7302 2.12639 15.8739C2.12598 16.0194 2.16368 16.1625 2.23574 16.2889C2.30779 16.4153 2.41169 16.5207 2.5371 16.5944C2.66141 16.6676 2.80256 16.7072 2.94673 16.7095H17.0527C17.1969 16.7072 17.338 16.6676 17.4624 16.5944C17.5878 16.5207 17.6917 16.4153 17.7637 16.2889C17.8358 16.1625 17.8735 16.0194 17.8731 15.8739C17.8727 15.7302 17.8351 15.5892 17.7642 15.4644L10.7122 3.69165C10.7119 3.6911 10.7116 3.69054 10.7112 3.68999C10.637 3.56828 10.5328 3.46763 10.4086 3.39769C10.2838 3.32742 10.143 3.29051 9.99973 3.29051ZM9.99973 6.70946C10.46 6.70946 10.8331 7.08256 10.8331 7.54279V10.8761C10.8331 11.3364 10.46 11.7095 9.99973 11.7095C9.53949 11.7095 9.1664 11.3364 9.1664 10.8761V7.54279C9.1664 7.08256 9.53949 6.70946 9.99973 6.70946ZM9.1664 14.2095C9.1664 13.7492 9.53949 13.3761 9.99973 13.3761H10.0081C10.4683 13.3761 10.8414 13.7492 10.8414 14.2095C10.8414 14.6697 10.4683 15.0428 10.0081 15.0428H9.99973C9.53949 15.0428 9.1664 14.6697 9.1664 14.2095Z" fill="#F79009"/>
+</svg>

packages/web/components/common/Icon/icons/common/confirm/restoreTip.svg

@@ -0,0 +1,3 @@
+<svg  viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M11.0058 3.68803C7.89786 3.68803 5.31329 5.93506 4.79046 8.89299L5.70669 8.3668C6.10579 8.13759 6.61514 8.27532 6.84434 8.67443C7.07355 9.07353 6.93582 9.58287 6.53671 9.81208L3.95412 11.2953C3.814 11.3757 3.66029 11.411 3.50989 11.4056C3.21062 11.4162 2.91562 11.2648 2.7564 10.9869L1.29256 8.43222C1.06374 8.03289 1.20197 7.52368 1.6013 7.29487C2.00063 7.06605 2.50984 7.20428 2.73865 7.60361L3.1897 8.39078C3.93457 4.75546 7.15021 2.02136 11.0058 2.02136C15.4123 2.02136 18.9844 5.59352 18.9844 9.99999C18.9844 14.4065 15.4123 17.9786 11.0058 17.9786C9.66714 17.9786 8.30206 17.5718 7.13895 16.9284C5.98005 16.2874 4.95794 15.3757 4.35999 14.3039C4.13578 13.902 4.27984 13.3944 4.68176 13.1701C5.08369 12.9459 5.59128 13.09 5.8155 13.4919C6.2256 14.2271 6.98588 14.9391 7.94567 15.47C8.90126 15.9986 9.9912 16.312 11.0058 16.312C14.4918 16.312 17.3178 13.486 17.3178 9.99999C17.3178 6.51399 14.4918 3.68803 11.0058 3.68803Z"/>
+</svg>

packages/web/components/common/Icon/icons/common/disable.svg

@@ -0,0 +1 @@
+<svg t="1740494996853" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="2899" width="64" height="64"><path d="M512 953.6a441.6 441.6 0 1 1 0-883.2 441.6 441.6 0 0 1 0 883.2z m0-64a377.6 377.6 0 1 0 0-755.2 377.6 377.6 0 0 0 0 755.2z"  p-id="2900"></path><path d="M182.1696 227.4304l45.2608-45.2608 614.4 614.4-45.2608 45.2608z"  p-id="2901"></path></svg>

packages/web/components/common/Icon/icons/common/enable.svg

@@ -0,0 +1 @@
+<svg t="1740495050372" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4745" width="64" height="64"><path d="M510.2 959.7c-246.9-1-447-202.6-446-449.5s202.6-447 449.5-446 447 202.6 446 449.5-202.6 447-449.5 446z m3.3-833.7c-212.8-0.8-386.7 171.7-387.5 384.5S297.7 897.2 510.5 898 897.2 726.3 898 513.5 726.3 126.8 513.5 126z" p-id="4746"></path><path d="M465.8 712.3L291.1 537.6l43.7-43.7 131 131 262-262 43.7 43.7z" p-id="4747"></path></svg>