4.6.8-alpha (#804)

* perf: redirect request and err log replace

perf: dataset openapi

feat: session

fix: retry input error

feat: 468 doc

sub page

feat: standard sub

perf: rerank tip

perf: rerank tip

perf: api sdk

perf: openapi

sub plan

perf: sub ui

fix: ts

* perf: init log

* fix: variable select

* sub page

* icon

* perf: llm model config

* perf: menu ux

* perf: system store

* perf: publish app name

* fix: init data

* perf: flow edit ux

* fix: value type format and ux

* fix prompt editor default value (#13)

* fix prompt editor default value

* fix prompt editor update when not focus

* add key with variable

---------

Co-authored-by: Archer <545436317@qq.com>

* fix: value type

* doc

* i18n

* import path

* home page

* perf: mongo session running

* fix: ts

* perf: use toast

* perf: flow edit

* perf: sse response

* slider ui

* fetch error

* fix prompt editor rerender when not focus by key defaultvalue (#14)

* perf: prompt editor

* feat: dataset search concat

* perf: doc

* fix:ts

* perf: doc

* fix json editor onblur value (#15)

* faq

* vector model default config

* ipv6

---------

Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

.gitignore

@@ -36,4 +36,7 @@ dist/
 docSite/public/
 docSite/resources/_gen/
 docSite/.vercel
-*.local.*
+*.local.*
+
+
+.idea/

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+  "recommendations": [
+    "inlang.vs-code-extension"
+  ]
+}

Dockerfile

@@ -78,7 +78,7 @@ COPY --from=builder /app/projects/$name/package.json ./package.json
 COPY --from=workerDeps /app/worker /app/worker
 # copy config
 COPY ./projects/$name/data /app/data
-
+RUN chown -R nextjs:nodejs /app/data
 
 ENV NODE_ENV production
 ENV NEXT_TELEMETRY_DISABLED 1

README.md

@@ -57,7 +57,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
    - [x] 源文件引用追踪
    - [x] 模块封装，实现多级复用
    - [x] 混合检索 & 重排
-   - [ ] 自查询规划
+   - [ ] Tool 模块
    - [ ] 嵌入 [Laf](https://github.com/labring/laf)，实现在线编写 HTTP 模块
    - [ ] 插件封装功能
 
@@ -67,10 +67,10 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
    - [x] 支持知识库单独设置向量模型
    - [x] 源文件存储
    - [x] 支持手动输入，直接分段，QA 拆分导入
-   - [x] 支持 pdf、word、txt、md 等常用文件，支持 url 读取、CSV 批量导入
+   - [x] 支持 pdf，docx，txt，html，md，csv
-   - [ ] 支持 HTML、csv、PPT、Excel 导入
+   - [x] 支持 url 读取、CSV 批量导入
+   - [ ] 支持 PPT、Excel 导入
    - [ ] 支持文件阅读器
-   - [ ] 支持差异性文件同步
    - [ ] 更多的数据预处理方案
   
 `3` 应用调试能力
@@ -81,8 +81,8 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
    - [ ] 高级编排 DeBug 模式
   
 `4` OpenAPI 接口
-   - [x] completions 接口 (对齐 GPT 接口)
+   - [x] completions 接口 (chat 模式对齐 GPT 接口)
-   - [ ] 知识库 CRUD
+   - [x] 知识库 CRUD
    - [ ] 对话 CRUD
   
 `5` 运营能力
@@ -215,4 +215,4 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 1. 允许作为后台服务直接商用，但不允许提供 SaaS 服务。
 2. 未经商业授权，任何形式的商用服务均需保留相关版权信息。
 3. 完整请查看 [FastGPT Open Source License](./LICENSE)
-4. 联系方式：yujinlong@sealos.io，[点击查看商业版定价策略](https://doc.fastgpt.in/docs/commercial)
+4. 联系方式：yujinlong@sealos.io，[点击查看商业版定价策略](https://doc.fastgpt.in/docs/commercial)

README_en.md

@@ -49,48 +49,52 @@ Cloud: [fastgpt.in](https://fastgpt.in/)
 
 ## 💡 Features
 
-1. Powerful visual workflows: Effortlessly craft AI applications
+`1` Application Orchestration Features
 
-   - [x] Simple mode on deck - no need for manual arrangement
+   - [x] Offers a straightforward mode, eliminating the need for complex orchestration
-   - [x] User dialogue pre-guidance
+   - [x] Provides clear next-step instructions in dialogues
-   - [x] Global variables
+   - [x] Facilitates workflow orchestration
-   - [x] Knowledge base search
+   - [x] Tracks references in source files
-   - [x] Dialogue via multiple LLM models
+   - [x] Encapsulates modules for enhanced reuse at multiple levels
-   - [x] Text magic - convert to structured data
+   - [x] Combines search and reordering functions
-   - [x] Extend with HTTP
+   - [ ] Includes a tool module
-   - [ ] Embed Laf for on-the-fly HTTP module crafting
+   - [ ] Integrates [Laf](https://github.com/labring/laf) for online HTTP module creation
-   - [x] Directions for the next dialogue steps
+   - [ ] Plugin encapsulation capabilities
-   - [x] Tracking source file references
-   - [ ] Custom file reader
-   - [ ] Modules are packaged into plug-ins to achieve reuse
 
-2. Extensive knowledge base preprocessing
+`2` Knowledge Base Features
 
-   - [x] Reuse and mix multiple knowledge bases
+   - [x] Allows for the mixed use of multiple databases
-   - [x] Track chunk modifications and deletions
+   - [x] Keeps track of modifications and deletions in data chunks
-   - [x] Supports manual entries, direct segmentation, and QA split imports
+   - [x] Enables specific vector models for each knowledge base
-   - [x] Supports URL fetching and batch CSV imports
+   - [x] Stores original source files
-   - [x] Supports Set unique vector models for knowledge bases
+   - [x] Supports direct input and segment-based QA import
-   - [x] Store original files
+   - [x] Compatible with a variety of file formats: pdf, docx, txt, html, md, csv
-   - [ ] File learning Agent
+   - [x] Facilitates URL reading and bulk CSV importing
+   - [ ] Supports PPT and Excel file import
+   - [ ] Features a file reader
+   - [ ] Offers diverse data preprocessing options
 
-3. Multiple effect testing channels
+`3` Application Debugging Features
 
-   - [x] Single-point knowledge base search test
+   - [x] Enables targeted search testing within the knowledge base
-   - [x] Feedback references and ability to modify and delete during dialogue
+   - [x] Allows feedback, editing, and deletion during conversations
-   - [x] Complete context presentation
+   - [x] Presents the full context of interactions
-   - [ ] Complete module intermediate value presentation
+   - [x] Displays all intermediate values within modules
+   - [ ] Advanced DeBug mode for orchestration
 
-4. OpenAPI
+`4` OpenAPI Interface
 
-   - [x] completions interface (aligned with GPT interface)
+   - [x] The completions interface (aligned with GPT's chat mode interface)
-   - [ ] Knowledge base CRUD
+   - [x] CRUD operations for the knowledge base
+   - [ ] CRUD operations for conversations
 
-5. Operational functions
+`5` Operational Features
+
+   - [x] Share without requiring login
+   - [x] Easy embedding with Iframe
+   - [x] Customizable chat window embedding with features like default open, drag-and-drop
+   - [x] Centralizes conversation records for review and annotation
 
-   - [x] Login-free sharing window
-   - [x] One-click embedding with Iframe
-   - [ ] Unified access to dialogue records
 
 <a href="#readme">
     <img src="https://img.shields.io/badge/-Back_to_Top-7d09f1.svg" alt="#" align="right">

dev.md

@@ -0,0 +1,17 @@
+# 打包命令
+
+```sh
+# Build image, not proxy
+docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app .
+
+# build image with proxy
+docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app --build-arg proxy=taobao .
+```
+
+# Pg 常用索引
+
+```sql
+CREATE INDEX IF NOT EXISTS modelData_dataset_id_index ON modeldata (dataset_id);
+CREATE INDEX IF NOT EXISTS modelData_collection_id_index ON modeldata (collection_id);
+CREATE INDEX IF NOT EXISTS modelData_teamId_index ON modeldata (team_id);
+```

docSite/content/docs/commercial/intro.md

@@ -44,15 +44,19 @@ FastGPT 商业版软件根据不同的部署方式，分为 3 类收费模式。
 **特有服务**
 
 {{< table "table-hover table-striped-columns" >}}
-| 部署方式 | 特有服务 | 上线时长 | 价格 |
+| 部署方式 | 特有服务 | 上线时长 | 标品价格 |
 | ---- | ---- | ---- | ---- |
 | Sealos全托管 |  1. 有效期内免费升级。<br>2. 免运维服务&数据库。 |  半天  | 3000元起/月（3个月起）<br>或<br>30000元起/年  |
 | 自有服务器-单机版 |  1. 6个版本的升级服务。 | 14天内 |  60000元/套（不限时长） | 
-| 自有服务器-Sealos版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）|
+| 自有服务器-高可用版 | 1. 6个版本的升级服务。 | 14天内 | 150000元/套（不限时长）|
 {{< /table >}}
 
 {{% alert icon="🤖 " context="success" %}}
-6个版本的升级服务不是指只能用 6 个版本，而是指依赖 FastGPT 团队提供的升级服务。大部分时候，建议自行升级，也不麻烦。
+- 6个版本的升级服务不是指只能用 6 个版本，而是指依赖 FastGPT 团队提供的升级服务。大部分时候，建议自行升级，也不麻烦。
+- 全托管版本适合技术人员紧缺的团队，仅需关注业务推动，无需关心服务是否正常运行。
+- 单机版和高可用版可以完全部署在自己服务器中。
+- 单机版适合中小团队对内提供服务，需要自己维护数据库备份等。
+- 高可用版适合对外提供在线服务，包含可视化监控、多副本、负载均衡、数据库自动备份等生产环境的基础设施。
 {{% /alert %}}
 
 

docSite/content/docs/course/data_search.md

@@ -29,7 +29,7 @@ weight: 106
 
 ### 全文检索
 
-才用传统的全文检索方式。适合查找关键的主谓语等。
+采用传统的全文检索方式。适合查找关键的主谓语等。
 
 ### 混合检索
 
@@ -55,4 +55,4 @@ FastGPT 会使用 `RRF` 对重排结果、向量搜索结果、全文检索结
 
 一个`0-1`的数值，会过滤掉一些低相关度的搜索结果。
 
-该值仅在`语义检索`或使用`结果重排`时生效。
+该值仅在`语义检索`或使用`结果重排`时生效。

docSite/content/docs/development/configuration.md

@@ -13,153 +13,7 @@ weight: 708
 
 这个配置文件中包含了系统级参数、AI 对话的模型、function 模型等……
 
-
+## 4.6.8 以前版本完整配置参数
-## 旧版本配置文件
-
-以下配置适合 4.6.6-alpha 之前
-
-```json
-{
-  "SystemParams": {
-    "vectorMaxProcess": 15, // 向量生成最大进程，结合数据库性能和 key 来设置
-    "qaMaxProcess": 15,  // QA 生成最大进程，结合数据库性能和 key 来设置
-    "pgHNSWEfSearch": 100  // pg vector 索引参数，越大精度高但速度慢
-  },
-  "ChatModels": [ // 对话模型
-    {
-      "model": "gpt-3.5-turbo-1106",
-      "name": "GPT35-1106",
-      "price": 0, // 除以 100000 后等于1个token的价格
-      "maxContext": 16000, // 最大上下文长度
-      "maxResponse": 4000, // 最大回复长度
-      "quoteMaxToken": 2000, // 最大引用内容长度
-      "maxTemperature": 1.2, // 最大温度值
-      "censor": false, // 是否开启敏感词过滤(商业版)
-      "vision": false, // 支持图片输入
-      "defaultSystemChatPrompt": ""
-    },
-    {
-      "model": "gpt-3.5-turbo-16k",
-      "name": "GPT35-16k",
-      "maxContext": 16000,
-      "maxResponse": 16000,
-      "price": 0,
-      "quoteMaxToken": 8000,
-      "maxTemperature": 1.2,
-      "censor": false,
-      "vision": false,
-      "defaultSystemChatPrompt": ""
-    },
-    {
-      "model": "gpt-4",
-      "name": "GPT4-8k",
-      "maxContext": 8000,
-      "maxResponse": 8000,
-      "price": 0,
-      "quoteMaxToken": 4000,
-      "maxTemperature": 1.2,
-      "censor": false,
-      "vision": false,
-      "defaultSystemChatPrompt": ""
-    },
-    {
-      "model": "gpt-4-vision-preview",
-      "name": "GPT4-Vision",
-      "maxContext": 128000,
-      "maxResponse": 4000,
-      "price": 0,
-      "quoteMaxToken": 100000,
-      "maxTemperature": 1.2,
-      "censor": false,
-      "vision": true,
-      "defaultSystemChatPrompt": ""
-    }
-  ],
-  "QAModels": [ // QA 生成模型
-    {
-      "model": "gpt-3.5-turbo-16k",
-      "name": "GPT35-16k",
-      "maxContext": 16000,
-      "maxResponse": 16000,
-      "price": 0
-    }
-  ],
-  "CQModels": [ // 问题分类模型
-    {
-      "model": "gpt-3.5-turbo-1106",
-      "name": "GPT35-1106",
-      "maxContext": 16000,
-      "maxResponse": 4000,
-      "price": 0,
-      "toolChoice": true, // 是否支持openai的 toolChoice， 不支持的模型需要设置为 false，会走提示词生成
-      "functionPrompt": ""
-    },
-    {
-      "model": "gpt-4",
-      "name": "GPT4-8k",
-      "maxContext": 8000,
-      "maxResponse": 8000,
-      "price": 0,
-      "toolChoice": true,
-      "functionPrompt": ""
-    }
-  ],
-  "ExtractModels": [ // 内容提取模型
-    {
-      "model": "gpt-3.5-turbo-1106",
-      "name": "GPT35-1106",
-      "maxContext": 16000,
-      "maxResponse": 4000,
-      "price": 0,
-      "toolChoice": true,
-      "functionPrompt": ""
-    }
-  ],
-  "QGModels": [ // 生成下一步指引
-    {
-      "model": "gpt-3.5-turbo-1106",
-      "name": "GPT35-1106",
-      "maxContext": 1600,
-      "maxResponse": 4000,
-      "price": 0
-    }
-  ],
-  "VectorModels": [ // 向量模型
-    {
-      "model": "text-embedding-ada-002",
-      "name": "Embedding-2",
-      "price": 0.2,
-      "defaultToken": 700,
-      "maxToken": 3000
-    }
-  ],
-  "ReRankModels": [], // 重排模型,暂时填空数组
-  "AudioSpeechModels": [
-    {
-      "model": "tts-1",
-      "name": "OpenAI TTS1",
-      "price": 0,
-      "baseUrl": "",
-      "key": "",
-      "voices": [
-        { "label": "Alloy", "value": "alloy", "bufferId": "openai-Alloy" },
-        { "label": "Echo", "value": "echo", "bufferId": "openai-Echo" },
-        { "label": "Fable", "value": "fable", "bufferId": "openai-Fable" },
-        { "label": "Onyx", "value": "onyx", "bufferId": "openai-Onyx" },
-        { "label": "Nova", "value": "nova", "bufferId": "openai-Nova" },
-        { "label": "Shimmer", "value": "shimmer", "bufferId": "openai-Shimmer" }
-      ]
-    }
-  ],
-  "WhisperModel": {
-    "model": "whisper-1",
-    "name": "Whisper1",
-    "price": 0
-  }
-}
-```
-
-## 4.6.6-alpha 版本完整配置参数
 
 **使用时，请务必去除注释！**
 
@@ -277,7 +131,7 @@ weight: 708
       "maxContext": 1600,
       "maxResponse": 4000,
       "inputPrice": 0,
-      "outputPrice": 0,
+      "outputPrice": 0
     }
   ],
   "vectorModels": [ // 向量模型
@@ -315,6 +169,134 @@ weight: 708
 }
 ```
 
+## 4.6.8 新配置文件
+
+llm模型全部合并
+
+```json
+{
+  "systemEnv": {
+    "openapiPrefix": "fastgpt",
+    "vectorMaxProcess": 15,
+    "qaMaxProcess": 15,
+    "pgHNSWEfSearch": 100
+  },
+  "llmModels": [
+    {
+      "model": "gpt-3.5-turbo-1106", // 模型名
+      "name": "gpt-3.5-turbo", // 别名
+      "maxContext": 16000, // 最大上下文
+      "maxResponse": 4000, // 最大回复
+      "quoteMaxToken": 13000, // 最大引用内容
+      "maxTemperature": 1.2, // 最大温度
+      "inputPrice": 0, 
+      "outputPrice": 0,
+      "censor": false,
+      "vision": false, // 是否支持图片输入
+      "datasetProcess": false, // 是否设置为知识库处理模型
+      "toolChoice": true, // 是否支持工具选择
+      "functionCall": false, // 是否支持函数调用
+      "customCQPrompt": "", // 自定义文本分类提示词（不支持工具和函数调用的模型
+      "customExtractPrompt": "", // 自定义内容提取提示词
+      "defaultSystemChatPrompt": "", // 对话默认携带的系统提示词
+      "defaultConfig":{}  // 对话默认配置（比如 GLM4 的 top_p
+    },
+    {
+      "model": "gpt-3.5-turbo-16k",
+      "name": "gpt-3.5-turbo-16k",
+      "maxContext": 16000,
+      "maxResponse": 16000,
+      "quoteMaxToken": 13000,
+      "maxTemperature": 1.2,
+      "inputPrice": 0,
+      "outputPrice": 0,
+      "censor": false,
+      "vision": false,
+      "datasetProcess": true,
+      "toolChoice": true,
+      "functionCall": false,
+      "customCQPrompt": "",
+      "customExtractPrompt": "",
+      "defaultSystemChatPrompt": "",
+      "defaultConfig":{} 
+    },
+    {
+      "model": "gpt-4-0125-preview",
+      "name": "gpt-4-turbo",
+      "maxContext": 125000,
+      "maxResponse": 125000,
+      "quoteMaxToken": 100000,
+      "maxTemperature": 1.2,
+      "inputPrice": 0,
+      "outputPrice": 0,
+      "censor": false,
+      "vision": false,
+      "datasetProcess": false,
+      "toolChoice": true,
+      "functionCall": false,
+      "customCQPrompt": "",
+      "customExtractPrompt": "",
+      "defaultSystemChatPrompt": "",
+      "defaultConfig":{} 
+    },
+    {
+      "model": "gpt-4-vision-preview",
+      "name": "gpt-4-vision",
+      "maxContext": 128000,
+      "maxResponse": 4000,
+      "quoteMaxToken": 100000,
+      "maxTemperature": 1.2,
+      "inputPrice": 0,
+      "outputPrice": 0,
+      "censor": false,
+      "vision": false,
+      "datasetProcess": false,
+      "toolChoice": true,
+      "functionCall": false,
+      "customCQPrompt": "",
+      "customExtractPrompt": "",
+      "defaultSystemChatPrompt": "",
+      "defaultConfig":{} 
+    }
+  ],
+  "vectorModels": [
+    {
+      "model": "text-embedding-ada-002",
+      "name": "Embedding-2",
+      "inputPrice": 0,
+      "outputPrice": 0,
+      "defaultToken": 700,
+      "maxToken": 3000,
+      "weight": 100,
+      "defaultConfig":{}  // 默认配置。例如，如果希望使用 embedding3-large 的话，可以传入 dimensions:1024，来返回1024维度的向量。（目前必须小于1536维度）
+    }
+  ],
+  "reRankModels": [],
+  "audioSpeechModels": [
+    {
+      "model": "tts-1",
+      "name": "OpenAI TTS1",
+      "inputPrice": 0,
+      "outputPrice": 0,
+      "voices": [
+        { "label": "Alloy", "value": "alloy", "bufferId": "openai-Alloy" },
+        { "label": "Echo", "value": "echo", "bufferId": "openai-Echo" },
+        { "label": "Fable", "value": "fable", "bufferId": "openai-Fable" },
+        { "label": "Onyx", "value": "onyx", "bufferId": "openai-Onyx" },
+        { "label": "Nova", "value": "nova", "bufferId": "openai-Nova" },
+        { "label": "Shimmer", "value": "shimmer", "bufferId": "openai-Shimmer" }
+      ]
+    }
+  ],
+  "whisperModel": {
+    "model": "whisper-1",
+    "name": "Whisper1",
+    "inputPrice": 0,
+    "outputPrice": 0
+  }
+}
+```
+
 ## 特殊模型
 
 ### ReRank 接入

docSite/content/docs/development/custom-models/chatglm2-m3e.md

@@ -59,10 +59,10 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
 
 ## 接入 FastGPT
 
-修改 config.json 配置文件，在 ChatModels 中加入 chatglm2, 在 VectorModels 中加入 M3E 模型：
+修改 config.json 配置文件，在 llmModels 中加入 chatglm2, 在 vectorModels 中加入 M3E 模型：
 
 ```json
-"ChatModels": [
+"llmModels": [
   //其他对话模型
   {
     "model": "chatglm2",
@@ -74,7 +74,7 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
     "defaultSystemChatPrompt": ""
   }
 ],
-"VectorModels": [
+"vectorModels": [
     {
       "model": "text-embedding-ada-002",
       "name": "Embedding-2",

docSite/content/docs/development/custom-models/chatglm2.md

@@ -99,10 +99,10 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
 
 ## 接入 FastGPT
 
-修改 config.json 配置文件，在 ChatModels 中加入 chatglm2 模型：
+修改 config.json 配置文件，在 llmModels 中加入 chatglm2 模型：
 
 ```json
-"ChatModels": [
+"llmModels": [
   //已有模型
   {
     "model": "chatglm2",

docSite/content/docs/development/custom-models/reranker.md

@@ -28,7 +28,7 @@ weight: 910
 ### 源码部署
 
 1. 根据上面的环境配置配置好环境，具体教程自行 GPT；
-2. 下载 [python 文件](app.py)
+2. 下载 [python 文件](https://github.com/labring/FastGPT/tree/main/python/reranker/bge-reranker-base)
 3. 在命令行输入命令 `pip install -r requirments.txt`；
 4. 按照[https://huggingface.co/BAAI/bge-reranker-base](https://huggingface.co/BAAI/bge-reranker-base)下载模型仓库到app.py同级目录
 5. 添加环境变量 `export ACCESS_TOKEN=XXXXXX` 配置 token，这里的 token 只是加一层验证，防止接口被人盗用，默认值为 `ACCESS_TOKEN` ；

docSite/content/docs/development/docker.md

@@ -17,6 +17,11 @@ weight: 707
 | 500w 组向量 | 8c32g | 16c64g 200GB |
 {{< /table >}}
 
+## 部署架构图
+
+![](/imgs/sealos-fastgpt.webp)
+
+
 ### 1. 准备好代理环境（国外服务器可忽略）
 
 确保可以访问 OpenAI，具体方案可以参考：[代理方案](/docs/development/proxy/)。或直接在 Sealos 上 [部署 OneAPI](/docs/development/one-api)，既解决代理问题也能实现多 Key 轮询、接入其他大模型。
@@ -89,10 +94,14 @@ curl -O https://raw.githubusercontent.com/labring/FastGPT/main/files/deploy/fast
 curl -O https://raw.githubusercontent.com/labring/FastGPT/main/projects/app/data/config.json
 ```
 
+## 三、修改 docker-compose.yml 的环境变量
 
-## 三、启动容器
+修改`docker-compose.yml`中的`OPENAI_BASE_URL`（API 接口的地址，需要加/v1）和`CHAT_API_KEY`（API 接口的凭证）。
 
-修改`docker-compose.yml`中的`OPENAI_BASE_URL`和`CHAT_API_KEY`即可，对应为 API 的地址(别忘记加/v1)和 key。
+使用 OneAPI 的话，OPENAI_BASE_URL=OneAPI访问地址/v1；CHAT_API_KEY=令牌
+
+
+## 四、启动容器
 
 ```bash
 # 在 docker-compose.yml 同级目录下执行
@@ -100,7 +109,39 @@ docker-compose pull
 docker-compose up -d
 ```
 
-## 四、访问 FastGPT
+## 四、初始化 Mongo 副本集(4.6.8以前可忽略)
+
+FastGPT 4.6.8 后使用了 MongoDB 的事务，需要运行在副本集上。副本集没法自动化初始化，需手动操作。
+
+```bash
+# 查看 mongo 容器是否正常运行
+docker ps 
+# 进入容器
+docker exec -it mongo bash
+
+# 连接数据库
+mongo
+
+# 初始化副本集。
+rs.initiate({
+  _id: "rs0",
+  members: [
+    { _id: 0, host: "mongo:27017" }
+  ]
+})
+# 检查状态。如果提示 rs0 状态，则代表运行成功
+rs.status()
+
+# 初始化用户
+use admin
+db.createUser({
+  user: "admin",
+  pwd: "password",
+  roles: [{ role: "root", db: "admin" }]
+});
+```
+
+## 五、访问 FastGPT
 
 目前可以通过 `ip:3000` 直接访问(注意防火墙)。登录用户名为 `root`，密码为`docker-compose.yml`环境变量里设置的 `DEFAULT_ROOT_PSW`。
 

docSite/content/docs/development/faq.md

@@ -19,13 +19,17 @@ images: []
 
 ## 通用问题
 
+### 能否纯本地允许
+
+可以。需要准备好向量模型和LLM模型。
+
 ### insufficient_user_quota user quota is not enough 
 
 OneAPI 账号的余额不足，默认 root 用户只有 200 刀，可以手动修改。
 
 ### xxx渠道找不到
 
-OneAPI 中没有配置该模型渠道。
+OneAPI 中没有配置该模型渠道。或者是修改了配置文件中一部分的模型，但没有全部修改。
 
 ### 页面中可以正常回复，API 报错
 
@@ -35,6 +39,25 @@ OneAPI 中没有配置该模型渠道。
 
 OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并重启容器（先 stop 然后 rm 掉，最后再 up -d 运行一次）。可以`exec`进入容器，`env`查看环境变量是否生效。
 
+### 其他模型没法进行问题分类/内容提取
+
+需要给其他模型配置`toolChoice=false`，就会默认走提示词模式。目前内置提示词仅针对了商业模型API进行测试，国内外的商业模型基本都可用。
+   
+### 页面崩溃
+
+1. 关闭翻译
+2. 检查配置文件是否正常加载，如果没有正常加载会导致缺失系统信息，在某些操作下会导致空指针。
+
+## 私有部署问题
+
+### 知识库索引没有进度
+
+先看日志报错信息。
+
+1. 可以对话，但是索引没有进度：没有配置向量模型（vectorModels）
+2. 不能对话，也不能索引：API调用失败。可能是没连上OneAPI或OenAI
+3. 有进度，但是非常慢：api key不行，OpenAI的免费号，一分钟只有3次还是60次。一天上限200次。
+
 ## Docker 部署常见问题
 
 ### 如何更新？
@@ -96,7 +119,7 @@ mongo连接失败，检查
 
 ### TypeError: Cannot read properties of null (reading 'useMemo' )
 
-用 Node18 试试，可能最新的 Node 有问题。 本地开发流程：
+删除所有的`node_modules`，用 Node18 重新 install 试试，可能最新的 Node 有问题。 本地开发流程：
 
 1. 根目录: `pnpm i`
 2. 复制 `config.json` -> `config.local.json`

docSite/content/docs/development/intro.md

@@ -48,6 +48,8 @@ git clone git@github.com:<github_username>/FastGPT.git
 
 第一次开发，需要先部署数据库，建议本地开发可以随便找一台 2C2G 的轻量小数据库实践。数据库部署教程：[Docker 快速部署](/docs/development/docker/)。部署完了，可以本地访问其数据库。
 
+Mongo 数据库需要修改副本集的`host`，从原来的`mongo:27017`修改为`ip:27017`。
+
 ### 4. 初始配置
 
 以下文件均在 `projects/app` 路径下。
@@ -62,7 +64,7 @@ git clone git@github.com:<github_username>/FastGPT.git
 
 **注意：json 配置文件不能包含注释，介绍中为了方便看才加入的注释**
 
-这个文件大部分时候不需要修改。只需要关注 SystemParams 里的参数：
+这个文件大部分时候不需要修改。只需要关注 `systemEnv` 里的参数：
 
 - `vectorMaxProcess`: 向量生成最大进程，根据数据库和 key 的并发数来决定，通常单个 120 号，2c4g 服务器设置 10~15。
 - `qaMaxProcess`: QA 生成最大进程

docSite/content/docs/development/one-api.md

@@ -11,6 +11,10 @@ weight: 708
 * [One API](https://github.com/songquanpeng/one-api) 是一个 OpenAI 接口管理 & 分发系统，可以通过标准的 OpenAI API 格式访问所有的大模型，开箱即用。
 * FastGPT 可以通过接入 OneAPI 来实现对不同大模型的支持。OneAPI 的部署方法也很简单。
 
+## FastGPT 与 OneAPI 关系
+
+![](/imgs/sealos-fastgpt.webp)
+
 ## MySQL 版本
 
 MySQL 版本支持多实例，高并发。
@@ -50,7 +54,14 @@ BATCH_UPDATE_ENABLED=true
 BATCH_UPDATE_INTERVAL=60
 ```
 
-## One API使用步骤
+## One API使用教程
+
+### 概念
+
+1. 渠道：
+   1. OneApi 中一个渠道对应一个 `Api Key`，这个 `Api Key` 可以是GPT、微软、ChatGLM、文心一言的。一个`Api Key`通常可以调用同一个厂商的多个模型。
+   2. OneAPI 会根据请求传入的`模型`来决定使用哪一个`Key`，如果一个模型对应了多个`Key`，则会随机调用。
+2. 令牌：访问 OneAPI 所需的凭证，只需要这`1`个凭证即可访问`OneAPI`上配置的模型。因此`FastGPT`中，只需要配置`OneAPI`的`baseurl`和`令牌`即可。
 
 ### 1. 登录 One API
 
@@ -68,7 +79,11 @@ BATCH_UPDATE_INTERVAL=60
 创建一个令牌
 ![step7](/imgs/oneapi-step7.png)
 
-### 3. 修改 FastGPT 的环境变量
+### 3. 修改账号余额
+
+OneAPI 默认 root 用户只有 200刀，可以自行修改编辑。
+
+### 4. 修改 FastGPT 的环境变量
 
 有了 One API 令牌后，FastGPT 可以通过修改 `baseurl` 和 `key` 去请求到 One API，再由 One API 去请求不同的模型。修改下面两个环境变量：
 
@@ -92,21 +107,29 @@ CHAT_API_KEY=sk-xxxxxx
 可以在 `/projects/app/src/data/config.json` 里找到配置文件（本地开发需要复制成 config.local.json），配置文件中有一项是对话模型配置：
 
 ```json
-"ChatModels": [
+"llmModels": [
     ...
     {
       "model": "ERNIE-Bot", // 这里的模型需要对应 One API 的模型
       "name": "文心一言", // 对外展示的名称
-      "maxContext": 8000, // 最大长下文 token，无论什么模型都按 GPT35 的计算。GPT 外的模型需要自行大致计算下这个值。可以调用官方接口去比对 Token 的倍率，然后在这里粗略计算。
+      "maxContext": 16000, // 最大上下文
-      "maxResponse": 4000, // 最大回复 token
+      "maxResponse": 4000, // 最大回复
-      // 例如：文心一言的中英文 token 基本是 1:1，而 GPT 的中文 Token 是 2:1，如果文心一言官方最大 Token 是 4000，那么这里就可以填 8000，保险点就填 7000.
+      "quoteMaxToken": 13000, // 最大引用内容
-      "quoteMaxToken": 2000, // 引用知识库的最大 Token
+      "maxTemperature": 1.2, // 最大温度
-      "maxTemperature": 1, // 最大温度
+      "inputPrice": 0, 
-      "vision": false, // 是否开启图片识别
+      "outputPrice": 0,
-      "defaultSystemChatPrompt": "" // 默认的系统提示词
+      "censor": false,
+      "vision": false, // 是否支持图片输入
+      "datasetProcess": false, // 是否设置为知识库处理模型
+      "toolChoice": true, // 是否支持工具选择
+      "functionCall": false, // 是否支持函数调用
+      "customCQPrompt": "", // 自定义文本分类提示词（不支持工具和函数调用的模型
+      "customExtractPrompt": "", // 自定义内容提取提示词
+      "defaultSystemChatPrompt": "", // 对话默认携带的系统提示词
+      "defaultConfig":{}  // 对话默认配置（比如 GLM4 的 top_p
     }
     ...
 ],
 ```
 
-添加完后，重启 FastGPT 即可在选择文心一言模型进行对话。
+添加完后，重启 FastGPT 即可在选择文心一言模型进行对话。**添加向量模型也是类似操作，增加到 `vectorModels`里。**

docSite/content/docs/development/openapi/chat.md

@@ -48,7 +48,7 @@ curl --location --request POST 'https://api.fastgpt.in/api/v1/chat/completions'
 {{< /markdownify >}}
 {{< /tab >}}
 
-{{< tab tabName="detail=true 响应" >}}
+{{< tab tabName="参数说明" >}}
 {{< markdownify >}}
 
 {{% alert context="info" %}}
@@ -56,7 +56,7 @@ curl --location --request POST 'https://api.fastgpt.in/api/v1/chat/completions'
 - chatId: string | undefined 。
   - 为 `undefined` 时（不传入），不使用 FastGpt 提供的上下文功能，完全通过传入的 messages 构建上下文。 不会将你的记录存储到数据库中，你也无法在记录汇总中查阅到。
   - 为`非空字符串`时，意味着使用 chatId 进行对话，自动从 FastGpt 数据库取历史记录，并使用 messages 数组最后一个内容作为用户问题。请自行确保 chatId 唯一，长度小于250，通常可以是自己系统的对话框ID。
-- messages: 结构与 [GPT接口](https://platform.openai.com/docs/api-reference/chat/object) 完全一致。
+- messages: 结构与 [GPT接口](https://platform.openai.com/docs/api-reference/chat/object) chat模式一致。
 - detail: 是否返回中间值（模块状态，响应的完整结果等），`stream模式`下会通过`event`进行区分，`非stream模式`结果保存在`responseData`中。
 - variables: 模块变量，一个对象，会替换模块中，输入框内容里的`{{key}}`
 {{% /alert %}}

docSite/content/docs/development/sealos.md

@@ -11,6 +11,13 @@ weight: 706
 
 ![](/imgs/sealos-fastgpt.webp)
 
+## 多模型支持
+
+FastGPT 使用了 one-api 项目来管理模型池，其可以兼容 OpenAI 、Azure 、国内主流模型和本地模型等。
+
+可参考：[Sealos 快速部署 OneAPI](/docs/development/one-api)
+
+
 ## 一键部署
 Sealos 的服务器在国外，不需要额外处理网络问题，无需服务器、无需魔法、无需域名，支持高并发 & 动态伸缩。点击以下按钮即可一键部署 👇
 

docSite/content/docs/development/upgrading/467.md

@@ -0,0 +1,35 @@
+---
+title: 'V4.6.7（需要初始化）'
+description: 'FastGPT V4.6.7'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 829
+---
+
+## 1。执行初始化 API
+
+发起 1 个 HTTP 请求 ({{rootkey}} 替换成环境变量里的 `rootkey`，{{host}} 替换成自己域名)
+
+1. https://xxxxx/api/admin/initv464
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv467' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化说明：
+1. 将 images 重新关联到数据集
+2. 设置 pg 表的 null 值。
+
+
+## V4.6.7 更新说明
+
+1. 修改了知识库UI及新的导入交互方式。
+2. 优化知识库和对话的数据索引。
+3. 知识库 openAPI，支持通过 [API 操作知识库](/docs/development/openapi/dataset)。
+4. 新增 - 输入框变量提示。输入 { 号后将会获得可用变量提示。根据社区针对高级编排的反馈，我们计划于 2 月份的版本中，优化变量内容，支持模块的局部变量以及更多全局变量写入。
+5. 优化 - 切换团队后会保存记录，下次登录时优先登录该团队。
+6. 修复 - API 对话时，chatId 冲突问题。
+7. 修复 - Iframe 嵌入网页可能导致的 window.onLoad 冲突。

docSite/content/docs/development/upgrading/468.md

@@ -0,0 +1,27 @@
+---
+title: 'V4.6.8（进行中）'
+description: 'FastGPT V4.6.7'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 828
+---
+
+## docker 部署 - 更新 Mongo
+
+开启 Mongo 副本集模式。需要进入 mongo 执行一次 init，参考[初始化Mongo副本集](/docs/development/docker/#四初始化-mongo-副本集)，这个比较麻烦，初始化后可以用 mongoshell 之类的连接试试，看能不能连接上。
+
+## Sealos 部署 - 无需更新 Mongo
+
+## 修改配置文件
+
+去除了重复的模型配置，LLM模型都合并到一个属性中：[点击查看最新的配置文件](/docs/development/configuration/)
+
+## V4.6.8 更新说明
+
+1. 新增 - 知识库搜索合并模块。
+1. 优化 - LLM 模型配置，不再区分对话、分类、提取模型。同时支持模型的默认参数，避免不同模型参数冲突，可通过`defaultConfig`传入默认的配置。
+2. 优化 - HTTP 模块，支持输出字符串自动序列化（JSON可自动转成字符串）
+3. 优化 - 流响应，参考了`ChatNextWeb`的流，更加丝滑。此外，之前提到的乱码、中断，刷新后又正常了，可能会修复）
+4. 修复 - 语音输入文件无法上传。
+5. 修复 - 对话框重新生成无法使用。

docSite/content/docs/use-cases/datasetEngine.md

@@ -25,7 +25,9 @@ FastGPT 采用了 RAG 中的 Embedding 方案构建知识库，要使用好 Fast
 
 FastGPT 采用了 `PostgresSQL` 的 `PG Vector` 插件作为向量检索器，索引为`HNSW`。且`PostgresSQL`仅用于向量检索，`MongoDB`用于其他数据的存取。
 
-在`PostgresSQL`的表中，设置一个 `index` 字段用于存储向量，以及一个`data_id`用于在`MongoDB`中寻找对应的映射值。多个`index`可以对应一组`data_id`，也就是说，一组向量可以对应多组数据。在进行检索时，相同数据会进行合并。
+在`MongoDB`的`dataset.datas`表中，会存储向量原数据的信息，同时有一个`indexes`字段，会记录其对应的向量ID，这是一个数组，也就是说，一组向量可以对应多组数据。
+
+在`PostgresSQL`的表中，设置一个 `index` 字段用于存储向量。在检索时，会先召回向量，再根据向量的ID，去`MongoDB`中寻找原数据内容，如果对应了同一组原数据，则进行合并，向量得分取最高得分。
 
 ![](/imgs/datasetSetting1.png)
 

docSite/content/docs/workflow/intro.md

@@ -30,20 +30,15 @@ FastGPT 从 V4 版本开始采用新的交互方式来构建 AI 应用。使用
 
 ### 模块分类
 
-从功能上，模块可以分为 3 类：
+从功能上，模块可以分为 2 类：
 
-1. **只读模块**：全局变量、用户引导。
+1. **系统模块**：用户引导（配置一些对话框信息）、用户问题（流程入口）。
-2. **系统模块**：聊天记录（无输入，直接从数据库取）、用户问题（流程入口）。
+2. **功能模块**：知识库搜索、AI 对话等剩余模块。（这些模块都有输入和输出，可以自由组合）。
-3. **功能模块**：知识库搜索、AI 对话等剩余模块。（这些模块都有输入和输出，可以自由组合）。
 
 ### 模块的组成
 
 每个模块会包含 3 个核心部分：固定参数、外部输入（左边有个圆圈）和输出（右边有个圆圈）。
 
-+ 对于只读模块，只需要根据提示填写即可，不参与流程运行。
-+ 对于系统模块，通常只有固定参数和输出，主要需要关注输出到哪个位置。
-+ 对于功能模块，通常这 3 部分都是重要的，以下图的 AI 对话为例：
-
    ![](/imgs/flow-intro3.png)
    
    - 对话模型、温度、回复上限、系统提示词和限定词为固定参数，同时系统提示词和限定词也可以作为外部输入，意味着如果你有输入流向了系统提示词，那么原本填写的内容就会被**覆盖**。
@@ -56,6 +51,7 @@ FastGPT 从 V4 版本开始采用新的交互方式来构建 AI 应用。使用
 
 1. 仅关心**已连接的**外部输入，即左边的圆圈被连接了。
 2. 当连接内容都有值时触发。
+3. **可以多个输出连接到一个输入，后续的值会覆盖前面的值。**
 
 #### 示例 1：
 
@@ -86,4 +82,17 @@ FastGPT 从 V4 版本开始采用新的交互方式来构建 AI 应用。使用
 
 1. 建议从左往右阅读。
 2. 从 **用户问题** 模块开始。用户问题模块，代表的是用户发送了一段文本，触发任务开始。
-3. 关注【AI 对话】和【指定回复】模块，这两个模块是输出答案的地方。
+3. 关注【AI 对话】和【指定回复】模块，这两个模块是输出答案的地方。
+
+## FAQ
+
+### 想合并多个输出结果怎么实现？
+
+1. 文本加工，可以对字符串进行合并。
+2. 知识库搜索合并，可以合并多个知识库搜索结果
+3. 其他结果，无法直接合并，可以考虑传入到`HTTP`模块中进行合并，使用`[Laf](https://laf.run/)`可以快速实现一个无服务器HTTP接口。
+
+### 模块为什么有2个用户问题
+
+左侧的`用户问题`是指该模块所需的输入。右侧的`用户问题`是为了方便后续的连线，输出的值和传入的用户问题一样。
+

docSite/content/docs/workflow/modules/http.md

@@ -38,7 +38,7 @@ HTTP 模块会向对应的地址发送一个 `POST/GET` 请求，携带部分`
 
 ### 嵌套对象使用
 
-**入参**
+#### 入参
 
 假设我们设计了`3个`输入。
 
@@ -58,7 +58,7 @@ HTTP 模块会向对应的地址发送一个 `POST/GET` 请求，携带部分`
 }
 ```
 
-**出参**
+#### 出参
 
 假设接口的输出结构为: 
 
@@ -66,18 +66,46 @@ HTTP 模块会向对应的地址发送一个 `POST/GET` 请求，携带部分`
 {
   "message": "测试",
   "data":{
-    "name": "name",
+      "user": {
-    "age": 10
+        "name": "xxx",
+        "age": 12
+      },
+      "list": [
+        {
+          "name": "xxx",
+          "age": 50
+        },
+        [{ "test": 22 }]
+      ],
+      "psw": "xxx"
   }
 }
 ```
 
-那么，自定出参的`key`可以设置为: 
+最终得到的解析为: 
 
-- message (string)
+```json
-- data.name (string)
+{
-- data.age (number)
+  "user": { "name": "xxx", "age": 12 },
+  "user.name": "xxx",
+  "user.age": 12,
+  "list": [ { "name": "xxx", "age": 50 }, [{ "test": 22 }] ],
+  "list[0]": { "name": "xxx", "age": 50 },
+  "list[0].name": "xxx",
+  "list[0].age": 50,
+  "list[1]": [ { "test": 22 } ],
+  "list[1][0]": { "test": 22 },
+  "list[1][0].test": 22,
+  "psw": "xxx"
+}
+```
 
+你可以使用`json`里对应的`key`来获取值。
+
+
+### 格式化输出
+
+FastGPT v4.6.8 后，加入了出参格式化功能，主要以`json`格式化成`字符串`为主。如果你的输出类型选择了`字符串`，则会将`HTTP`对应`key`的值，转成`json`字符串进行输出。因此，未来你可以直接从`HTTP`接口输出内容至`文本加工`中，然后拼接适当的提示词，最终输入给`AI对话`。
 
 ## POST 示例
 

files/deploy/fastgpt/clash/config.yaml

@@ -1,43 +0,0 @@
-mixed-port: 7890
-allow-lan: false
-bind-address: '*'
-mode: rule
-log-level: warning
-dns:
-  enable: true
-  ipv6: false
-  nameserver:
-    - 8.8.8.8
-    - 8.8.4.4
-  cache-size: 400
-proxies:
-
-proxy-groups:
-  - {
-      name: '♻️ 自动选择',
-      type: url-test,
-      proxies:
-        [
-          香港V02×1.5,
-          ABC,
-          印度01,
-          台湾03,
-          新加坡02,
-          新加坡03,
-          日本01,
-          日本02,
-          新加坡01,
-          美国01,
-          美国02,
-          台湾01,
-          台湾02
-        ],
-      url: 'https://api.openai.com',
-      interval: 3600
-    }
-rules:
-  - 'DOMAIN-SUFFIX,google.com,♻️ 自动选择'
-  - 'DOMAIN-SUFFIX,ai.fastgpt.in,♻️ 自动选择'
-  - 'DOMAIN-SUFFIX,openai.com,♻️ 自动选择'
-  - 'DOMAIN-SUFFIX,api.openai.com,♻️ 自动选择'
-  - 'MATCH,DIRECT'

files/deploy/fastgpt/clash/proxy.sh

@@ -1,18 +0,0 @@
-export ALL_PROXY=socks5://127.0.0.1:7891
-export http_proxy=http://127.0.0.1:7890
-export https_proxy=http://127.0.0.1:7890
-export HTTP_PROXY=http://127.0.0.1:7890
-export HTTPS_PROXY=http://127.0.0.1:7890
-
-OLD_PROCESS=$(pgrep clash)
-if [ ! -z "$OLD_PROCESS" ]; then
-  echo "Killing old process: $OLD_PROCESS"
-  kill $OLD_PROCESS
-fi
-sleep 2
-
-cd  /root/fastgpt/clash/fast
-rm -f ./nohup.out || true
-rm -f ./cache.db || true
-nohup ./clash-linux-amd64-v3  -d ./ &
-echo "Restart clash fast"

files/deploy/fastgpt/clash/stop.sh

@@ -1,10 +0,0 @@
-export ALL_PROXY=''
-export http_proxy=''
-export https_proxy=''
-export HTTP_PROXY=''
-export HTTPS_PROXY=''
-OLD_PROCESS=$(pgrep clash)
-if [ ! -z "$OLD_PROCESS" ]; then
-  echo "Killing old process: $OLD_PROCESS"
-  kill $OLD_PROCESS
-fi

files/deploy/fastgpt/docker-compose.yml

@@ -27,10 +27,7 @@ services:
       - 27017:27017
     networks:
       - fastgpt
-    environment:
+    command: --replSet rs0
-      # 这里的配置只有首次运行生效。修改后，重启镜像是不会生效的。需要把持久化数据删除再重启，才有效果
-      - MONGO_INITDB_ROOT_USERNAME=username
-      - MONGO_INITDB_ROOT_PASSWORD=password
     volumes:
       - ./mongo/data:/data/db
   fastgpt:
@@ -55,63 +52,11 @@ services:
       - TOKEN_KEY=any
       - ROOT_KEY=root_key
       - FILE_TOKEN_KEY=filetoken
-      # mongo 配置，不需要改. 如果连不上，可能需要去掉 ?authSource=admin
+      # mongo 配置，不需要改. 用户名admin,密码password，是执行 db.createUser 时的账号和密码。
-      - MONGODB_URI=mongodb://username:password@mongo:27017/fastgpt?authSource=admin
+      - MONGODB_URI=mongodb://admin:password@mongo:27017/fastgpt?authSource=admin
       # pg配置. 不需要改
       - PG_URL=postgresql://username:password@pg:5432/postgres
     volumes:
       - ./config.json:/app/data/config.json
 networks:
   fastgpt:
-# host 版本, 不推荐。
-# version: '3.3'
-# services:
-#   pg:
-#   image: ankane/pgvector:v0.5.0 # dockerhub
-#   # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.5.0 # 阿里云
-#   container_name: pg
-#   restart: always
-#   ports: # 生产环境建议不要暴露
-#       - 5432:5432
-#   environment:
-#       # 这里的配置只有首次运行生效。修改后，重启镜像是不会生效的。需要把持久化数据删除再重启，才有效果
-#       - POSTGRES_USER=username
-#       - POSTGRES_PASSWORD=password
-#       - POSTGRES_DB=postgres
-#   volumes:
-#       - ./pg/data:/var/lib/postgresql/data
-#   mongo:
-#   image: mongo:5.0.18
-#   # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/mongo:5.0.18 # 阿里云
-#   container_name: mongo
-#   restart: always
-#   ports: # 生产环境建议不要暴露
-#       - 27017:27017
-#   environment:
-#       # 这里的配置只有首次运行生效。修改后，重启镜像是不会生效的。需要把持久化数据删除再重启，才有效果
-#       - MONGO_INITDB_ROOT_USERNAME=username
-#       - MONGO_INITDB_ROOT_PASSWORD=password
-#   volumes:
-#       - ./mongo/data:/data/db
-#       - ./mongo/logs:/var/log/mongodb
-#   fastgpt:
-#   # image: ghcr.io/labring/fastgpt:latest # github
-#   image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:latest # 阿里云
-#   network_mode: host
-#   restart: always
-#   container_name: fastgpt
-#   environment:
-#       # root 密码，用户名为: root
-#       - DEFAULT_ROOT_PSW=1234
-#       # 中转地址，如果是用官方号，不需要管
-#       - OPENAI_BASE_URL=https://api.openai.com/v1
-#       - CHAT_API_KEY=sk-xxxx
-#       - DB_MAX_LINK=5 # database max link
-#       # token加密凭证（随便填，作为登录凭证）
-#       - TOKEN_KEY=any
-#       # root key, 最高权限，可以内部接口互相调用
-#       - ROOT_KEY=root_key
-#       # mongo 配置，不需要改
-#       - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/fastgpt?authSource=admin
-#       # pg配置. 不需要改
-#       - PG_URL=postgresql://username:password@0.0.0.0:5432/postgres

packages/global/common/error/utils.ts

@@ -1,5 +1,7 @@
+import { replaceSensitiveLink } from '../string/tools';
+
 export const getErrText = (err: any, def = '') => {
   const msg: string = typeof err === 'string' ? err : err?.message || def || '';
   msg && console.log('error =>', msg);
-  return msg;
+  return replaceSensitiveLink(msg);
 };

packages/global/common/file/api.d.ts

@@ -1,9 +1,15 @@
-export type UploadImgProps = {
+import { MongoImageTypeEnum } from './image/constants';
-  base64Img: string;
+
+export type preUploadImgProps = {
+  type: `${MongoImageTypeEnum}`;
+
   expiredTime?: Date;
   metadata?: Record<string, any>;
   shareId?: string;
 };
+export type UploadImgProps = preUploadImgProps & {
+  base64Img: string;
+};
 
 export type UrlFetchParams = {
   urlList: string[];
@@ -11,6 +17,7 @@ export type UrlFetchParams = {
 };
 export type UrlFetchResponse = {
   url: string;
+  title: string;
   content: string;
   selector?: string;
 }[];

packages/global/common/file/icon.ts

@@ -1,12 +1,14 @@
 export const fileImgs = [
-  { suffix: 'pdf', src: '/imgs/files/pdf.svg' },
+  { suffix: 'pdf', src: 'file/fill/pdf' },
-  { suffix: 'csv', src: '/imgs/files/csv.svg' },
+  { suffix: 'csv', src: 'file/fill/csv' },
-  { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' },
+  { suffix: '(doc|docs)', src: 'file/fill/doc' },
-  { suffix: 'txt', src: '/imgs/files/txt.svg' },
+  { suffix: 'txt', src: 'file/fill/txt' },
-  { suffix: 'md', src: '/imgs/files/markdown.svg' }
+  { suffix: 'md', src: 'file/fill/markdown' },
+  { suffix: 'html', src: 'file/fill/html' }
+
   // { suffix: '.', src: '/imgs/files/file.svg' }
 ];
 
-export function getFileIcon(name = '', defaultImg = '/imgs/files/file.svg') {
+export function getFileIcon(name = '', defaultImg = 'file/fill/file') {
   return fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src || defaultImg;
 }

packages/global/common/file/image/constants.ts

@@ -0,0 +1,52 @@
+export const imageBaseUrl = '/api/system/img/';
+
+export enum MongoImageTypeEnum {
+  systemAvatar = 'systemAvatar',
+  appAvatar = 'appAvatar',
+  pluginAvatar = 'pluginAvatar',
+  datasetAvatar = 'datasetAvatar',
+  userAvatar = 'userAvatar',
+  teamAvatar = 'teamAvatar',
+
+  chatImage = 'chatImage',
+  collectionImage = 'collectionImage'
+}
+export const mongoImageTypeMap = {
+  [MongoImageTypeEnum.systemAvatar]: {
+    label: 'common.file.type.appAvatar',
+    unique: true
+  },
+  [MongoImageTypeEnum.appAvatar]: {
+    label: 'common.file.type.appAvatar',
+    unique: true
+  },
+  [MongoImageTypeEnum.pluginAvatar]: {
+    label: 'common.file.type.pluginAvatar',
+    unique: true
+  },
+  [MongoImageTypeEnum.datasetAvatar]: {
+    label: 'common.file.type.datasetAvatar',
+    unique: true
+  },
+  [MongoImageTypeEnum.userAvatar]: {
+    label: 'common.file.type.userAvatar',
+    unique: true
+  },
+  [MongoImageTypeEnum.teamAvatar]: {
+    label: 'common.file.type.teamAvatar',
+    unique: true
+  },
+
+  [MongoImageTypeEnum.chatImage]: {
+    label: 'common.file.type.chatImage',
+    unique: false
+  },
+  [MongoImageTypeEnum.collectionImage]: {
+    label: 'common.file.type.collectionImage',
+    unique: false
+  }
+};
+
+export const uniqueImageTypeList = Object.entries(mongoImageTypeMap)
+  .filter(([key, value]) => value.unique)
+  .map(([key]) => key as `${MongoImageTypeEnum}`);

packages/global/common/file/image/type.d.ts

@@ -0,0 +1,14 @@
+import { MongoImageTypeEnum } from './constants';
+
+export type MongoImageSchemaType = {
+  _id: string;
+  teamId: string;
+  binary: Buffer;
+  createTime: Date;
+  expiredTime?: Date;
+  type: `${MongoImageTypeEnum}`;
+
+  metadata?: {
+    relatedId?: string; // This id is associated with a set of images
+  };
+};

packages/global/common/math/date.ts

@@ -0,0 +1,18 @@
+// The number of days left in the month is calculated as 30 days per month, and less than 1 day is calculated as 1 day
+export const getMonthRemainingDays = (startDate = new Date()) => {
+  const year = startDate.getFullYear();
+  const month = startDate.getMonth();
+  const endDay = new Date(year, month + 1, 0, 0, 0, 0);
+  return calculateDaysBetweenDates(startDate, endDay);
+};
+
+export const calculateDaysBetweenDates = (date1: Date, date2: Date) => {
+  const oneDay = 24 * 60 * 60 * 1000;
+  const firstDate = new Date(date1).getTime();
+  const secondDate = new Date(date2).getTime();
+
+  const differenceInTime = Math.abs(secondDate - firstDate);
+  const differenceInDays = Math.floor(differenceInTime / oneDay);
+
+  return differenceInDays;
+};

packages/global/common/string/markdown.ts

@@ -15,10 +15,10 @@ export const simpleMarkdownText = (rawText: string) => {
     return `[${cleanedLinkText}](${url})`;
   });
 
-  // replace special \.* ……
+  // replace special #\.* ……
-  const reg1 = /\\([-.!`_(){}\[\]])/g;
+  const reg1 = /\\([#`!*()+-_\[\]{}\\.])/g;
   if (reg1.test(rawText)) {
-    rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1');
+    rawText = rawText.replace(reg1, '$1');
   }
 
   // replace \\n
@@ -45,14 +45,15 @@ export const uploadMarkdownBase64 = async ({
   uploadImgController
 }: {
   rawText: string;
-  uploadImgController: (base64: string) => Promise<string>;
+  uploadImgController?: (base64: string) => Promise<string>;
 }) => {
-  // match base64, upload and replace it
+  if (uploadImgController) {
-  const base64Regex = /data:image\/.*;base64,([^\)]+)/g;
+    // match base64, upload and replace it
-  const base64Arr = rawText.match(base64Regex) || [];
+    const base64Regex = /data:image\/.*;base64,([^\)]+)/g;
-  // upload base64 and replace it
+    const base64Arr = rawText.match(base64Regex) || [];
-  await Promise.all(
+
-    base64Arr.map(async (base64Img) => {
+    // upload base64 and replace it
+    for await (const base64Img of base64Arr) {
       try {
         const str = await uploadImgController(base64Img);
 
@@ -61,8 +62,8 @@ export const uploadMarkdownBase64 = async ({
         rawText = rawText.replace(base64Img, '');
         rawText = rawText.replace(/!\[.*\]\(\)/g, '');
       }
-    })
+    }
-  );
+  }
 
   // Remove white space on both sides of the picture
   const trimReg = /(!\[.*\]\(.*\))\s*/g;
@@ -70,5 +71,20 @@ export const uploadMarkdownBase64 = async ({
     rawText = rawText.replace(trimReg, '$1');
   }
 
-  return simpleMarkdownText(rawText);
+  return rawText;
+};
+
+export const markdownProcess = async ({
+  rawText,
+  uploadImgController
+}: {
+  rawText: string;
+  uploadImgController?: (base64: string) => Promise<string>;
+}) => {
+  const imageProcess = await uploadMarkdownBase64({
+    rawText,
+    uploadImgController
+  });
+
+  return simpleMarkdownText(imageProcess);
 };

packages/global/common/string/textSplitter.ts

@@ -13,13 +13,12 @@ export const splitText2Chunks = (props: {
   chunkLen: number;
   overlapRatio?: number;
   customReg?: string[];
-  countTokens?: boolean;
 }): {
   chunks: string[];
-  tokens: number;
+  chars: number;
   overlapRatio?: number;
 } => {
-  let { text = '', chunkLen, overlapRatio = 0.2, customReg = [], countTokens = true } = props;
+  let { text = '', chunkLen, overlapRatio = 0.2, customReg = [] } = props;
   const splitMarker = 'SPLIT_HERE_SPLIT_HERE';
   const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER';
   const overlapLen = Math.round(chunkLen * overlapRatio);
@@ -240,13 +239,11 @@ export const splitText2Chunks = (props: {
       mdTitle: ''
     }).map((chunk) => chunk?.replaceAll(codeBlockMarker, '\n') || ''); // restore code block
 
-    const tokens = countTokens
+    const chars = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
-      ? chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0)
-      : 0;
 
     return {
       chunks,
-      tokens
+      chars
     };
   } catch (err) {
     throw new Error(getErrText(err));

packages/global/common/string/tiktoken/index.ts

@@ -33,6 +33,12 @@ export function countPromptTokens(
 ) {
   const enc = getTikTokenEnc();
   const text = `${role}\n${prompt}`;
+
+  // too large a text will block the thread
+  if (text.length > 15000) {
+    return text.length * 1.7;
+  }
+
   try {
     const encodeText = enc.encode(text);
     return encodeText.length + role.length; // 补充 role 估算值

packages/global/common/string/time.ts

@@ -1,3 +1,4 @@
 import dayjs from 'dayjs';
 
-export const formatTime2YMDHM = (time: Date) => dayjs(time).format('YYYY-MM-DD HH:mm');
+export const formatTime2YMDHM = (time?: Date) =>
+  time ? dayjs(time).format('YYYY-MM-DD HH:mm') : '';

packages/global/common/string/tools.ts

@@ -1,4 +1,5 @@
 import crypto from 'crypto';
+import { customAlphabet } from 'nanoid';
 
 /* check string is a web link */
 export function strIsLink(str?: string) {
@@ -36,3 +37,13 @@ export function replaceVariable(text: string, obj: Record<string, string | numbe
   }
   return text || '';
 }
+
+/* replace sensitive link */
+export const replaceSensitiveLink = (text: string) => {
+  const urlRegex = /(?<=https?:\/\/)[^\s]+/g;
+  return text.replace(urlRegex, 'xxx');
+};
+
+export const getNanoid = (size = 12) => {
+  return customAlphabet('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', size)();
+};

packages/global/common/system/types/index.d.ts

@@ -1,3 +1,4 @@
+import { StandSubPlanLevelMapType, SubPlanType } from '../../../support/wallet/sub/type';
 import type {
   ChatModelItemType,
   FunctionModelItemType,
@@ -7,16 +8,14 @@ import type {
   WhisperModelType,
   ReRankModelItemType
 } from '../../../core/ai/model.d';
+import { SubTypeEnum } from '../../../support/wallet/sub/constants';
 
 /* fastgpt main */
 export type FastGPTConfigFileType = {
   feConfigs: FastGPTFeConfigsType;
   systemEnv: SystemEnvType;
-  chatModels: ChatModelItemType[];
+  subPlans?: SubPlanType;
-  qaModels: LLMModelItemType[];
+  llmModels: ChatModelItemType[];
-  cqModels: FunctionModelItemType[];
-  extractModels: FunctionModelItemType[];
-  qgModels: LLMModelItemType[];
   vectorModels: VectorModelItemType[];
   reRankModels: ReRankModelItemType[];
   audioSpeechModels: AudioSpeechModelType[];
@@ -51,6 +50,8 @@ export type FastGPTFeConfigsType = {
   favicon?: string;
   customApiDomain?: string;
   customSharePageDomain?: string;
+
+  uploadFileMaxSize?: number;
 };
 
 export type SystemEnvType = {
@@ -60,7 +61,8 @@ export type SystemEnvType = {
   pgHNSWEfSearch: number;
 };
 
-declare global {
+// declare global {
-  var feConfigs: FastGPTFeConfigsType;
+//   var feConfigs: FastGPTFeConfigsType;
-  var systemEnv: SystemEnvType;
+//   var systemEnv: SystemEnvType;
-}
+//   var systemInitd: boolean;
+// }

packages/global/core/ai/model.d.ts

@@ -3,20 +3,24 @@ export type LLMModelItemType = {
   name: string;
   maxContext: number;
   maxResponse: number;
-  inputPrice: number;
-  outputPrice: number;
-};
-export type ChatModelItemType = LLMModelItemType & {
   quoteMaxToken: number;
   maxTemperature: number;
+
+  inputPrice: number;
+  outputPrice: number;
+
   censor?: boolean;
   vision?: boolean;
-  defaultSystemChatPrompt?: string;
+  datasetProcess?: boolean;
-};
 
-export type FunctionModelItemType = LLMModelItemType & {
+  functionCall: boolean;
   toolChoice: boolean;
-  functionPrompt: string;
+
+  customCQPrompt: string;
+  customExtractPrompt: string;
+
+  defaultSystemChatPrompt?: string;
+  defaultConfig?: Record<string, any>;
 };
 
 export type VectorModelItemType = {
@@ -27,6 +31,8 @@ export type VectorModelItemType = {
   outputPrice: number;
   maxToken: number;
   weight: number;
+  hidden?: boolean;
+  defaultConfig?: Record<string, any>;
 };
 
 export type ReRankModelItemType = {

packages/global/core/ai/model.ts

@@ -3,11 +3,22 @@ import type { LLMModelItemType, VectorModelItemType } from './model.d';
 export const defaultQAModels: LLMModelItemType[] = [
   {
     model: 'gpt-3.5-turbo-16k',
-    name: 'GPT35-16k',
+    name: 'gpt-3.5-turbo-16k',
     maxContext: 16000,
     maxResponse: 16000,
+    quoteMaxToken: 13000,
+    maxTemperature: 1.2,
     inputPrice: 0,
-    outputPrice: 0
+    outputPrice: 0,
+    censor: false,
+    vision: false,
+    datasetProcess: true,
+    toolChoice: true,
+    functionCall: false,
+    customCQPrompt: '',
+    customExtractPrompt: '',
+    defaultSystemChatPrompt: '',
+    defaultConfig: {}
   }
 ];
 

packages/global/core/app/api.d.ts

@@ -1,4 +1,4 @@
-import type { ChatModelItemType } from '../ai/model.d';
+import type { LLMModelItemType } from '../ai/model.d';
 import { AppTypeEnum } from './constants';
 import { AppSchema, AppSimpleEditFormType } from './type';
 
@@ -22,5 +22,5 @@ export interface AppUpdateParams {
 export type FormatForm2ModulesProps = {
   formData: AppSimpleEditFormType;
   chatModelMaxToken: number;
-  chatModelList: ChatModelItemType[];
+  llmModelList: LLMModelItemType[];
 };

packages/global/core/app/type.d.ts

@@ -4,7 +4,7 @@ import { PermissionTypeEnum } from '../../support/permission/constant';
 import type { AIChatModuleProps, DatasetModuleProps } from '../module/node/type.d';
 import { VariableInputEnum } from '../module/constants';
 import { SelectedDatasetType } from '../module/api';
-import { DatasetSearchModeEnum } from '../dataset/constant';
+import { DatasetSearchModeEnum } from '../dataset/constants';
 
 export interface AppSchema {
   _id: string;

packages/global/core/app/utils.ts

@@ -4,7 +4,7 @@ import { ModuleOutputKeyEnum, ModuleInputKeyEnum } from '../module/constants';
 import type { FlowNodeInputItemType } from '../module/node/type.d';
 import { getGuideModule, splitGuideModule } from '../module/utils';
 import { ModuleItemType } from '../module/type.d';
-import { DatasetSearchModeEnum } from '../dataset/constant';
+import { DatasetSearchModeEnum } from '../dataset/constants';
 
 export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEditFormType => {
   return {
@@ -91,7 +91,7 @@ export const appModules2Form = ({
       );
       defaultAppForm.dataset.limit = findInputValueByKey(
         module.inputs,
-        ModuleInputKeyEnum.datasetLimit
+        ModuleInputKeyEnum.datasetMaxTokens
       );
       defaultAppForm.dataset.searchMode =
         findInputValueByKey(module.inputs, ModuleInputKeyEnum.datasetSearchMode) ||

packages/global/core/chat/constants.ts

@@ -31,16 +31,16 @@ export enum ChatSourceEnum {
 }
 export const ChatSourceMap = {
   [ChatSourceEnum.test]: {
-    name: 'chat.logs.test'
+    name: 'core.chat.logs.test'
   },
   [ChatSourceEnum.online]: {
-    name: 'chat.logs.online'
+    name: 'core.chat.logs.online'
   },
   [ChatSourceEnum.share]: {
-    name: 'chat.logs.share'
+    name: 'core.chat.logs.share'
   },
   [ChatSourceEnum.api]: {
-    name: 'chat.logs.api'
+    name: 'core.chat.logs.api'
   }
 };
 

packages/global/core/chat/type.d.ts

@@ -4,7 +4,7 @@ import { ChatRoleEnum, ChatSourceEnum, ChatStatusEnum } from './constants';
 import { FlowNodeTypeEnum } from '../module/node/constant';
 import { ModuleOutputKeyEnum } from '../module/constants';
 import { AppSchema } from '../app/type';
-import { DatasetSearchModeEnum } from '../dataset/constant';
+import { DatasetSearchModeEnum } from '../dataset/constants';
 
 export type ChatSchema = {
   _id: string;
@@ -22,6 +22,7 @@ export type ChatSchema = {
   shareId?: string;
   outLinkUid?: string;
   content: ChatItemType[];
+  metadata?: Record<string, any>;
 };
 
 export type ChatWithAppSchema = Omit<ChatSchema, 'appId'> & {
@@ -91,6 +92,7 @@ export type moduleDispatchResType = {
   runningTime?: number;
   inputTokens?: number;
   outputTokens?: number;
+  charsLength?: number;
   model?: string;
   query?: string;
   contextTotalLen?: number;

packages/global/core/dataset/api.d.ts

@@ -1,5 +1,5 @@
 import { DatasetDataIndexItemType, DatasetSchemaType } from './type';
-import { DatasetCollectionTrainingModeEnum, DatasetCollectionTypeEnum } from './constant';
+import { TrainingModeEnum, DatasetCollectionTypeEnum } from './constants';
 import type { LLMModelItemType } from '../ai/model.d';
 
 /* ================= dataset ===================== */
@@ -16,28 +16,47 @@ export type DatasetUpdateBody = {
 };
 
 /* ================= collection ===================== */
-export type CreateDatasetCollectionParams = {
+export type DatasetCollectionChunkMetadataType = {
-  datasetId: string;
   parentId?: string;
+  trainingType?: `${TrainingModeEnum}`;
+  chunkSize?: number;
+  chunkSplitter?: string;
+  qaPrompt?: string;
+  metadata?: Record<string, any>;
+};
+export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
+  datasetId: string;
   name: string;
   type: `${DatasetCollectionTypeEnum}`;
-  trainingType?: `${DatasetCollectionTrainingModeEnum}`;
-  chunkSize?: number;
   fileId?: string;
   rawLink?: string;
-  qaPrompt?: string;
   rawTextLength?: number;
   hashRawText?: string;
-  metadata?: Record<string, any>;
+};
+
+export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
+  datasetId: string;
+};
+export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
+  name: string;
+  text: string;
+};
+export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
+  link: string;
+};
+export type FileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
+  name: string;
+  rawTextLength: number;
+  hashRawText: string;
+
+  fileMetadata?: Record<string, any>;
+  collectionMetadata?: Record<string, any>;
 };
 
 /* ================= data ===================== */
 export type PgSearchRawType = {
   id: string;
-  team_id: string;
-  tmb_id: string;
   collection_id: string;
-  data_id: string;
   score: number;
 };
 export type PushDatasetDataChunkProps = {
@@ -51,3 +70,14 @@ export type PostWebsiteSyncParams = {
   datasetId: string;
   billId: string;
 };
+
+export type PushDatasetDataProps = {
+  collectionId: string;
+  data: PushDatasetDataChunkProps[];
+  trainingMode: `${TrainingModeEnum}`;
+  prompt?: string;
+  billId?: string;
+};
+export type PushDatasetDataResponse = {
+  insertLen: number;
+};

packages/global/core/dataset/constants.ts

@@ -6,7 +6,7 @@ export enum DatasetTypeEnum {
 }
 export const DatasetTypeMap = {
   [DatasetTypeEnum.folder]: {
-    icon: 'core/dataset/folderDataset',
+    icon: 'common/folderFill',
     label: 'core.dataset.Folder Dataset',
     collectionLabel: 'common.Folder'
   },
@@ -53,23 +53,7 @@ export const DatasetCollectionTypeMap = {
     name: 'core.dataset.link'
   },
   [DatasetCollectionTypeEnum.virtual]: {
-    name: 'core.dataset.Virtual File'
+    name: 'core.dataset.Manual collection'
-  }
-};
-export enum DatasetCollectionTrainingModeEnum {
-  manual = 'manual',
-  chunk = 'chunk',
-  qa = 'qa'
-}
-export const DatasetCollectionTrainingTypeMap = {
-  [DatasetCollectionTrainingModeEnum.manual]: {
-    label: 'core.dataset.collection.training.type manual'
-  },
-  [DatasetCollectionTrainingModeEnum.chunk]: {
-    label: 'core.dataset.collection.training.type chunk'
-  },
-  [DatasetCollectionTrainingModeEnum.qa]: {
-    label: 'core.dataset.collection.training.type qa'
   }
 };
 
@@ -120,10 +104,12 @@ export enum TrainingModeEnum {
 
 export const TrainingTypeMap = {
   [TrainingModeEnum.chunk]: {
-    label: 'core.dataset.training.type chunk'
+    label: 'core.dataset.training.Chunk mode',
+    tooltip: 'core.dataset.import.Chunk Split Tip'
   },
   [TrainingModeEnum.qa]: {
-    label: 'core.dataset.training.type qa'
+    label: 'core.dataset.training.QA mode',
+    tooltip: 'core.dataset.import.QA Import Tip'
   }
 };
 
@@ -184,4 +170,8 @@ export const SearchScoreTypeMap = {
   }
 };
 
-export const FolderAvatarSrc = '/imgs/files/folder.svg';
+export const FolderIcon = 'file/fill/folder';
+export const FolderImgUrl = '/imgs/files/folder.svg';
+
+export const CustomCollectionIcon = 'common/linkBlue';
+export const LinkCollectionIcon = 'common/linkBlue';

packages/global/core/dataset/controller.d.ts

@@ -21,7 +21,7 @@ export type UpdateDatasetDataProps = {
 };
 
 export type PatchIndexesProps = {
-  type: 'create' | 'update' | 'delete';
+  type: 'create' | 'update' | 'delete' | 'unChange';
   index: Omit<DatasetDataIndexItemType, 'dataId'> & {
     dataId?: string;
   };

packages/global/core/dataset/search/utils.ts

@@ -0,0 +1,95 @@
+import { countPromptTokens } from '../../../common/string/tiktoken';
+import { SearchScoreTypeEnum } from '../constants';
+import { SearchDataResponseItemType } from '../type';
+
+/* dataset search result concat */
+export const datasetSearchResultConcat = (
+  arr: { k: number; list: SearchDataResponseItemType[] }[]
+): SearchDataResponseItemType[] => {
+  arr = arr.filter((item) => item.list.length > 0);
+
+  if (arr.length === 0) return [];
+  if (arr.length === 1) return arr[0].list;
+
+  const map = new Map<string, SearchDataResponseItemType & { rrfScore: number }>();
+
+  // rrf
+  arr.forEach((item) => {
+    const k = item.k;
+
+    item.list.forEach((data, index) => {
+      const rank = index + 1;
+      const score = 1 / (k + rank);
+
+      const record = map.get(data.id);
+      if (record) {
+        // 合并两个score,有相同type的score,取最大值
+        const concatScore = [...record.score];
+        for (const dataItem of data.score) {
+          const sameScore = concatScore.find((item) => item.type === dataItem.type);
+          if (sameScore) {
+            sameScore.value = Math.max(sameScore.value, dataItem.value);
+          } else {
+            concatScore.push(dataItem);
+          }
+        }
+
+        map.set(data.id, {
+          ...record,
+          score: concatScore,
+          rrfScore: record.rrfScore + score
+        });
+      } else {
+        map.set(data.id, {
+          ...data,
+          rrfScore: score
+        });
+      }
+    });
+  });
+
+  // sort
+  const mapArray = Array.from(map.values());
+  const results = mapArray.sort((a, b) => b.rrfScore - a.rrfScore);
+
+  return results.map((item, index) => {
+    // if SearchScoreTypeEnum.rrf exist, reset score
+    const rrfScore = item.score.find((item) => item.type === SearchScoreTypeEnum.rrf);
+    if (rrfScore) {
+      rrfScore.value = item.rrfScore;
+      rrfScore.index = index;
+    } else {
+      item.score.push({
+        type: SearchScoreTypeEnum.rrf,
+        value: item.rrfScore,
+        index
+      });
+    }
+
+    // @ts-ignore
+    delete item.rrfScore;
+    return item;
+  });
+};
+
+export const filterSearchResultsByMaxChars = (
+  list: SearchDataResponseItemType[],
+  maxTokens: number
+) => {
+  const results: SearchDataResponseItemType[] = [];
+  let totalTokens = 0;
+
+  for (let i = 0; i < list.length; i++) {
+    const item = list[i];
+    totalTokens += countPromptTokens(item.q + item.a);
+    if (totalTokens > maxTokens + 500) {
+      break;
+    }
+    results.push(item);
+    if (totalTokens > maxTokens) {
+      break;
+    }
+  }
+
+  return results.length === 0 ? list.slice(0, 1) : results;
+};

packages/global/core/dataset/type.d.ts

@@ -8,7 +8,7 @@ import {
   DatasetTypeEnum,
   SearchScoreTypeEnum,
   TrainingModeEnum
-} from './constant';
+} from './constants';
 
 /* schema */
 export type DatasetSchemaType = {
@@ -42,15 +42,21 @@ export type DatasetCollectionSchemaType = {
   type: `${DatasetCollectionTypeEnum}`;
   createTime: Date;
   updateTime: Date;
+
   trainingType: `${TrainingModeEnum}`;
   chunkSize: number;
+  chunkSplitter?: string;
+  qaPrompt?: string;
+
   fileId?: string;
   rawLink?: string;
-  qaPrompt?: string;
+
   rawTextLength?: number;
   hashRawText?: string;
   metadata?: {
     webPageSelector?: string;
+    relatedImgId?: string; // The id of the associated image collections
+
     [key: string]: any;
   };
 };

packages/global/core/dataset/utils.ts

@@ -1,4 +1,4 @@
-import { DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constant';
+import { TrainingModeEnum, DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constants';
 import { getFileIcon } from '../../common/file/icon';
 import { strIsLink } from '../../common/string/tools';
 
@@ -7,18 +7,13 @@ export function getCollectionIcon(
   name = ''
 ) {
   if (type === DatasetCollectionTypeEnum.folder) {
-    return '/imgs/files/folder.svg';
+    return 'common/folderFill';
   }
   if (type === DatasetCollectionTypeEnum.link) {
-    return '/imgs/files/link.svg';
+    return 'common/linkBlue';
   }
   if (type === DatasetCollectionTypeEnum.virtual) {
-    if (name === '手动录入') {
+    return 'file/fill/manual';
-      return '/imgs/files/manual.svg';
-    } else if (name === '手动标注') {
-      return '/imgs/files/mark.svg';
-    }
-    return '/imgs/files/collection.svg';
   }
   return getFileIcon(name);
 }
@@ -30,21 +25,17 @@ export function getSourceNameIcon({
   sourceId?: string;
 }) {
   if (strIsLink(sourceId)) {
-    return '/imgs/files/link.svg';
+    return 'common/linkBlue';
   }
   const fileIcon = getFileIcon(sourceName, '');
   if (fileIcon) {
     return fileIcon;
   }
 
-  if (sourceName === '手动录入') {
+  return 'file/fill/manual';
-    return '/imgs/files/manual.svg';
-  } else if (sourceName === '手动标注') {
-    return '/imgs/files/mark.svg';
-  }
-  return '/imgs/files/collection.svg';
 }
 
+/* get dataset data default index */
 export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: string }) {
   const { q = '', a, dataId } = props || {};
   const qaStr = `${q}\n${a}`.trim();
@@ -55,3 +46,8 @@ export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: strin
     dataId
   };
 }
+
+export const predictDataLimitLength = (mode: `${TrainingModeEnum}`, data: any[]) => {
+  if (mode === TrainingModeEnum.qa) return data.length * 20;
+  return data.length;
+};

packages/global/core/module/constants.ts

@@ -61,7 +61,7 @@ export enum ModuleInputKeyEnum {
   // dataset
   datasetSelectList = 'datasets',
   datasetSimilarity = 'similarity',
-  datasetLimit = 'limit',
+  datasetMaxTokens = 'limit',
   datasetSearchMode = 'searchMode',
   datasetSearchUsingReRank = 'usingReRank',
   datasetParamsModal = 'datasetParamsModal',
@@ -113,5 +113,16 @@ export enum VariableInputEnum {
   textarea = 'textarea',
   select = 'select'
 }
+export const variableMap = {
+  [VariableInputEnum.input]: {
+    icon: 'core/app/variable/input'
+  },
+  [VariableInputEnum.textarea]: {
+    icon: 'core/app/variable/textarea'
+  },
+  [VariableInputEnum.select]: {
+    icon: 'core/app/variable/select'
+  }
+};
 
 export const DYNAMIC_INPUT_KEY = 'DYNAMIC_INPUT_KEY';

packages/global/core/module/node/constant.ts

@@ -45,7 +45,10 @@ export enum FlowNodeTypeEnum {
   questionInput = 'questionInput',
   historyNode = 'historyNode',
   chatNode = 'chatNode',
+
   datasetSearchNode = 'datasetSearchNode',
+  datasetConcatNode = 'datasetConcatNode',
+
   answerNode = 'answerNode',
   classifyQuestion = 'classifyQuestion',
   contentExtract = 'contentExtract',
@@ -54,10 +57,9 @@ export enum FlowNodeTypeEnum {
   pluginModule = 'pluginModule',
   pluginInput = 'pluginInput',
   pluginOutput = 'pluginOutput',
-  cfr = 'cfr',
+  cfr = 'cfr'
 
   // abandon
-  variable = 'variable'
 }
 
 export const EDGE_TYPE = 'default';

packages/global/core/module/node/type.d.ts

@@ -106,6 +106,6 @@ export type AIChatModuleProps = {
 export type DatasetModuleProps = {
   [ModuleInputKeyEnum.datasetSelectList]: SelectedDatasetType;
   [ModuleInputKeyEnum.datasetSimilarity]: number;
-  [ModuleInputKeyEnum.datasetLimit]: number;
+  [ModuleInputKeyEnum.datasetMaxTokens]: number;
   [ModuleInputKeyEnum.datasetStartReRank]: boolean;
 };

packages/global/core/module/template/input.ts

@@ -7,6 +7,7 @@ export const Input_Template_Switch: FlowNodeInputItemType = {
   key: ModuleInputKeyEnum.switch,
   type: FlowNodeInputTypeEnum.target,
   label: 'core.module.input.label.switch',
+  description: 'core.module.input.description.Trigger',
   valueType: ModuleIOValueTypeEnum.any,
   showTargetInApp: true,
   showTargetInPlugin: true
@@ -27,8 +28,8 @@ export const Input_Template_History: FlowNodeInputItemType = {
 
 export const Input_Template_UserChatInput: FlowNodeInputItemType = {
   key: ModuleInputKeyEnum.userChatInput,
-  type: FlowNodeInputTypeEnum.hidden,
+  type: FlowNodeInputTypeEnum.custom,
-  label: 'core.module.input.label.user question',
+  label: '',
   required: true,
   valueType: ModuleIOValueTypeEnum.string,
   showTargetInApp: true,
@@ -56,3 +57,13 @@ export const Input_Template_DynamicInput: FlowNodeInputItemType = {
   showTargetInPlugin: true,
   hideInApp: true
 };
+
+export const Input_Template_Dataset_Quote: FlowNodeInputItemType = {
+  key: ModuleInputKeyEnum.aiChatDatasetQuote,
+  type: FlowNodeInputTypeEnum.target,
+  label: '知识库引用',
+  description: 'core.module.Dataset quote.Input description',
+  valueType: ModuleIOValueTypeEnum.datasetQuote,
+  showTargetInApp: true,
+  showTargetInPlugin: true
+};

packages/global/core/module/template/system/abandon/history.ts

@@ -1,52 +0,0 @@
-import {
-  FlowNodeInputTypeEnum,
-  FlowNodeOutputTypeEnum,
-  FlowNodeTypeEnum
-} from '../../../node/constant';
-import { FlowModuleTemplateType } from '../../../type';
-import {
-  ModuleIOValueTypeEnum,
-  ModuleInputKeyEnum,
-  ModuleTemplateTypeEnum
-} from '../../../constants';
-
-export const HistoryModule: FlowModuleTemplateType = {
-  id: FlowNodeTypeEnum.historyNode,
-  templateType: ModuleTemplateTypeEnum.systemInput,
-  flowType: FlowNodeTypeEnum.historyNode,
-  avatar: '/imgs/module/history.png',
-  name: '聊天记录(弃用)',
-  intro: '聊天记录，该模块已被弃用',
-  inputs: [
-    {
-      key: ModuleInputKeyEnum.historyMaxAmount,
-      type: FlowNodeInputTypeEnum.numberInput,
-      label: '最长记录数',
-      description:
-        '该记录数不代表模型可接收这么多的历史记录，具体可接收多少历史记录，取决于模型的能力，通常建议不要超过20条。',
-      value: 6,
-      valueType: ModuleIOValueTypeEnum.number,
-      min: 0,
-      max: 100,
-      showTargetInApp: false,
-      showTargetInPlugin: false
-    },
-    {
-      key: ModuleInputKeyEnum.history,
-      type: FlowNodeInputTypeEnum.hidden,
-      valueType: ModuleIOValueTypeEnum.chatHistory,
-      label: '聊天记录',
-      showTargetInApp: false,
-      showTargetInPlugin: false
-    }
-  ],
-  outputs: [
-    {
-      key: ModuleInputKeyEnum.history,
-      label: '聊天记录',
-      valueType: ModuleIOValueTypeEnum.chatHistory,
-      type: FlowNodeOutputTypeEnum.source,
-      targets: []
-    }
-  ]
-};

packages/global/core/module/template/system/aiChat.ts

@@ -11,6 +11,7 @@ import {
   ModuleTemplateTypeEnum
 } from '../../constants';
 import {
+  Input_Template_Dataset_Quote,
   Input_Template_History,
   Input_Template_Switch,
   Input_Template_UserChatInput
@@ -23,15 +24,15 @@ export const AiChatModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.textAnswer,
   flowType: FlowNodeTypeEnum.chatNode,
   avatar: '/imgs/module/AI.png',
-  name: 'AI 对话',
+  name: 'core.module.template.Ai chat',
-  intro: 'AI 大模型对话',
+  intro: 'core.module.template.Ai chat intro',
   showStatus: true,
   inputs: [
     Input_Template_Switch,
     {
       key: ModuleInputKeyEnum.aiModel,
       type: FlowNodeInputTypeEnum.selectChatModel,
-      label: '对话模型',
+      label: 'core.module.input.label.aiModel',
       required: true,
       valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
@@ -41,42 +42,31 @@ export const AiChatModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.aiChatTemperature,
       type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window
-      label: '温度',
+      label: '',
       value: 0,
       valueType: ModuleIOValueTypeEnum.number,
       min: 0,
       max: 10,
       step: 1,
-      markList: [
-        { label: '严谨', value: 0 },
-        { label: '发散', value: 10 }
-      ],
       showTargetInApp: false,
       showTargetInPlugin: false
     },
     {
       key: ModuleInputKeyEnum.aiChatMaxToken,
       type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window
-      label: '回复上限',
+      label: '',
       value: 2000,
       valueType: ModuleIOValueTypeEnum.number,
       min: 100,
       max: 4000,
       step: 50,
-      markList: [
-        { label: '100', value: 100 },
-        {
-          label: `${4000}`,
-          value: 4000
-        }
-      ],
       showTargetInApp: false,
       showTargetInPlugin: false
     },
     {
       key: ModuleInputKeyEnum.aiChatIsResponseText,
       type: FlowNodeInputTypeEnum.hidden,
-      label: '返回AI内容',
+      label: '',
       value: true,
       valueType: ModuleIOValueTypeEnum.boolean,
       showTargetInApp: false,
@@ -85,7 +75,7 @@ export const AiChatModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.aiChatQuoteTemplate,
       type: FlowNodeInputTypeEnum.hidden,
-      label: '引用内容模板',
+      label: '',
       valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
       showTargetInPlugin: false
@@ -93,7 +83,7 @@ export const AiChatModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.aiChatQuotePrompt,
       type: FlowNodeInputTypeEnum.hidden,
-      label: '引用内容提示词',
+      label: '',
       valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
       showTargetInPlugin: false
@@ -110,7 +100,7 @@ export const AiChatModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.aiSystemPrompt,
       type: FlowNodeInputTypeEnum.textarea,
-      label: '系统提示词',
+      label: 'core.ai.Prompt',
       max: 300,
       valueType: ModuleIOValueTypeEnum.string,
       description: chatNodeSystemPromptTip,
@@ -119,31 +109,23 @@ export const AiChatModule: FlowModuleTemplateType = {
       showTargetInPlugin: true
     },
     Input_Template_History,
-    {
+    Input_Template_UserChatInput,
-      key: ModuleInputKeyEnum.aiChatDatasetQuote,
+    Input_Template_Dataset_Quote
-      type: FlowNodeInputTypeEnum.target,
-      label: '引用内容',
-      description: "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
-      valueType: ModuleIOValueTypeEnum.datasetQuote,
-      showTargetInApp: true,
-      showTargetInPlugin: true
-    },
-    Input_Template_UserChatInput
   ],
   outputs: [
     Output_Template_UserChatInput,
     {
       key: ModuleOutputKeyEnum.history,
-      label: '新的上下文',
+      label: 'core.module.output.label.New context',
-      description: '将本次回复内容拼接上历史记录，作为新的上下文返回',
+      description: 'core.module.output.description.New context',
       valueType: ModuleIOValueTypeEnum.chatHistory,
       type: FlowNodeOutputTypeEnum.source,
       targets: []
     },
     {
       key: ModuleOutputKeyEnum.answerText,
-      label: 'AI回复内容',
+      label: 'core.module.output.label.Ai response content',
-      description: '将在 stream 回复完毕后触发',
+      description: 'core.module.output.description.Ai response content',
       valueType: ModuleIOValueTypeEnum.string,
       type: FlowNodeOutputTypeEnum.source,
       targets: []

packages/global/core/module/template/system/assignedAnswer.ts

@@ -9,19 +9,17 @@ export const AssignedAnswerModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.textAnswer,
   flowType: FlowNodeTypeEnum.answerNode,
   avatar: '/imgs/module/reply.png',
-  name: '指定回复',
+  name: 'core.module.template.Assigned reply',
-  intro: '该模块可以直接回复一段指定的内容。常用于引导、提示',
+  intro: 'core.module.template.Assigned reply intro',
   inputs: [
     Input_Template_Switch,
     {
       key: ModuleInputKeyEnum.answerText,
       type: FlowNodeInputTypeEnum.textarea,
       valueType: ModuleIOValueTypeEnum.any,
-      label: '回复的内容',
+      label: 'core.module.input.label.Response content',
-      description:
+      description: 'core.module.input.description.Response content',
-        '可以使用 \\n 来实现连续换行。\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n如传入非字符串类型数据将会自动转成字符串',
+      placeholder: 'core.module.input.description.Response content',
-      placeholder:
-        '可以使用 \\n 来实现连续换行。\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n如传入非字符串类型数据将会自动转成字符串',
       showTargetInApp: true,
       showTargetInPlugin: true
     }

packages/global/core/module/template/system/classifyQuestion.ts

@@ -17,12 +17,8 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.functionCall,
   flowType: FlowNodeTypeEnum.classifyQuestion,
   avatar: '/imgs/module/cq.png',
-  name: '问题分类',
+  name: 'core.module.template.Classify question',
-  intro: `根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：
+  intro: `core.module.template.Classify question intro`,
-类型1: 打招呼
-类型2: 关于商品“使用”问题
-类型3: 关于商品“购买”问题
-类型4: 其他问题`,
   showStatus: true,
   inputs: [
     Input_Template_Switch,
@@ -30,7 +26,7 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.aiModel,
       type: FlowNodeInputTypeEnum.selectCQModel,
       valueType: ModuleIOValueTypeEnum.string,
-      label: '分类模型',
+      label: 'core.module.input.label.Classify model',
       required: true,
       showTargetInApp: false,
       showTargetInPlugin: false
@@ -39,11 +35,9 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.aiSystemPrompt,
       type: FlowNodeInputTypeEnum.textarea,
       valueType: ModuleIOValueTypeEnum.string,
-      label: '背景知识',
+      label: 'core.module.input.label.Background',
-      description:
+      description: 'core.module.input.description.Background',
-        '你可以添加一些特定内容的介绍，从而更好的识别用户的问题类型。这个内容通常是给模型介绍一个它不知道的内容。',
+      placeholder: 'core.module.input.placeholder.Classify background',
-      placeholder:
-        '例如: \n1. AIGC（人工智能生成内容）是指使用人工智能技术自动或半自动地生成数字内容，如文本、图像、音乐、视频等。\n2. AIGC技术包括但不限于自然语言处理、计算机视觉、机器学习和深度学习。这些技术可以创建新内容或修改现有内容，以满足特定的创意、教育、娱乐或信息需求。',
       showTargetInApp: true,
       showTargetInPlugin: true
     },

packages/global/core/module/template/system/contextExtract.ts

@@ -17,8 +17,8 @@ export const ContextExtractModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.functionCall,
   flowType: FlowNodeTypeEnum.contentExtract,
   avatar: '/imgs/module/extract.png',
-  name: '文本内容提取',
+  name: 'core.module.template.Extract field',
-  intro: '可从文本中提取指定的数据，例如：sql语句、搜索关键词、代码等',
+  intro: 'core.module.template.Extract field intro',
   showStatus: true,
   inputs: [
     Input_Template_Switch,
@@ -26,7 +26,7 @@ export const ContextExtractModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.aiModel,
       type: FlowNodeInputTypeEnum.selectExtractModel,
       valueType: ModuleIOValueTypeEnum.string,
-      label: '提取模型',
+      label: 'core.module.input.label.LLM',
       required: true,
       showTargetInApp: false,
       showTargetInPlugin: false

packages/global/core/module/template/system/datasetConcat.ts

@@ -0,0 +1,54 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import {
+  ModuleIOValueTypeEnum,
+  ModuleInputKeyEnum,
+  ModuleOutputKeyEnum,
+  ModuleTemplateTypeEnum
+} from '../../constants';
+import { Input_Template_Dataset_Quote, Input_Template_Switch } from '../input';
+import { Output_Template_Finish } from '../output';
+import { getNanoid } from '../../../../common/string/tools';
+
+export const getOneQuoteInputTemplate = (key = getNanoid()) => ({
+  ...Input_Template_Dataset_Quote,
+  key,
+  type: FlowNodeInputTypeEnum.hidden
+});
+
+export const DatasetConcatModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.datasetConcatNode,
+  flowType: FlowNodeTypeEnum.datasetConcatNode,
+  templateType: ModuleTemplateTypeEnum.tools,
+  avatar: '/imgs/module/concat.svg',
+  name: '知识库搜索引用合并',
+  intro: 'core.module.template.Dataset search result concat intro',
+  showStatus: false,
+  inputs: [
+    Input_Template_Switch,
+    {
+      key: ModuleInputKeyEnum.datasetMaxTokens,
+      type: FlowNodeInputTypeEnum.custom,
+      label: '最大 Tokens',
+      value: 1500,
+      valueType: ModuleIOValueTypeEnum.number,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    getOneQuoteInputTemplate('defaultQuote')
+  ],
+  outputs: [
+    {
+      key: ModuleOutputKeyEnum.datasetQuoteQA,
+      label: 'core.module.Dataset quote.label',
+      type: FlowNodeOutputTypeEnum.source,
+      valueType: ModuleIOValueTypeEnum.datasetQuote,
+      targets: []
+    },
+    Output_Template_Finish
+  ]
+};

packages/global/core/module/template/system/datasetSearch.ts

@@ -12,22 +12,22 @@ import {
 } from '../../constants';
 import { Input_Template_Switch, Input_Template_UserChatInput } from '../input';
 import { Output_Template_Finish, Output_Template_UserChatInput } from '../output';
-import { DatasetSearchModeEnum } from '../../../dataset/constant';
+import { DatasetSearchModeEnum } from '../../../dataset/constants';
 
 export const DatasetSearchModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.datasetSearchNode,
   templateType: ModuleTemplateTypeEnum.functionCall,
   flowType: FlowNodeTypeEnum.datasetSearchNode,
   avatar: '/imgs/module/db.png',
-  name: '知识库搜索',
+  name: 'core.module.template.Dataset search',
-  intro: '去知识库中搜索对应的答案。可作为 AI 对话引用参考。',
+  intro: 'core.module.template.Dataset search intro',
   showStatus: true,
   inputs: [
     Input_Template_Switch,
     {
       key: ModuleInputKeyEnum.datasetSelectList,
       type: FlowNodeInputTypeEnum.selectDataset,
-      label: '关联的知识库',
+      label: 'core.module.input.label.Select dataset',
       value: [],
       valueType: ModuleIOValueTypeEnum.selectDataset,
       list: [],
@@ -38,7 +38,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.datasetSimilarity,
       type: FlowNodeInputTypeEnum.hidden,
-      label: '最低相关性',
+      label: '',
       value: 0.4,
       valueType: ModuleIOValueTypeEnum.number,
       min: 0,
@@ -52,10 +52,9 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       showTargetInPlugin: false
     },
     {
-      key: ModuleInputKeyEnum.datasetLimit,
+      key: ModuleInputKeyEnum.datasetMaxTokens,
       type: FlowNodeInputTypeEnum.hidden,
-      label: '引用上限',
+      label: '',
-      description: '单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens',
       value: 1500,
       valueType: ModuleIOValueTypeEnum.number,
       showTargetInApp: false,
@@ -93,23 +92,21 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
     Output_Template_UserChatInput,
     {
       key: ModuleOutputKeyEnum.datasetIsEmpty,
-      label: '搜索结果为空',
+      label: 'core.module.output.label.Search result empty',
       type: FlowNodeOutputTypeEnum.source,
       valueType: ModuleIOValueTypeEnum.boolean,
       targets: []
     },
     {
       key: ModuleOutputKeyEnum.datasetUnEmpty,
-      label: '搜索结果不为空',
+      label: 'core.module.output.label.Search result not empty',
       type: FlowNodeOutputTypeEnum.source,
       valueType: ModuleIOValueTypeEnum.boolean,
       targets: []
     },
     {
       key: ModuleOutputKeyEnum.datasetQuoteQA,
-      label: '引用内容',
+      label: 'core.module.Dataset quote.label',
-      description:
-        '始终返回数组，如果希望搜索结果为空时执行额外操作，需要用到上面的两个输入以及目标模块的触发器',
       type: FlowNodeOutputTypeEnum.source,
       valueType: ModuleIOValueTypeEnum.datasetQuote,
       targets: []

packages/global/core/module/template/system/http.ts

@@ -17,8 +17,8 @@ export const HttpModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.externalCall,
   flowType: FlowNodeTypeEnum.httpRequest,
   avatar: '/imgs/module/http.png',
-  name: 'HTTP模块',
+  name: 'core.module.template.Http request',
-  intro: '可以发出一个 HTTP POST 请求，实现更为复杂的操作（联网搜索、数据库查询等）',
+  intro: 'core.module.template.Http request intro',
   showStatus: true,
   inputs: [
     Input_Template_Switch,

packages/global/core/module/template/system/runApp.ts

@@ -22,8 +22,8 @@ export const RunAppModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.externalCall,
   flowType: FlowNodeTypeEnum.runApp,
   avatar: '/imgs/module/app.png',
-  name: '应用调用',
+  name: 'core.module.template.Running app',
-  intro: '可以选择一个其他应用进行调用',
+  intro: 'core.module.template.Running app intro',
   showStatus: true,
   inputs: [
     Input_Template_Switch,

packages/global/core/module/template/system/runPlugin.ts

@@ -6,9 +6,9 @@ export const RunPluginModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.pluginModule,
   templateType: ModuleTemplateTypeEnum.externalCall,
   flowType: FlowNodeTypeEnum.pluginModule,
-  avatar: '/imgs/module/custom.png',
+  avatar: '',
   intro: '',
-  name: '自定义模块',
+  name: '',
   showStatus: false,
   inputs: [], // [{key:'pluginId'},...]
   outputs: []

packages/global/core/module/template/system/userGuide.ts

@@ -8,14 +8,14 @@ export const UserGuideModule: FlowModuleTemplateType = {
   templateType: ModuleTemplateTypeEnum.userGuide,
   flowType: FlowNodeTypeEnum.userGuide,
   avatar: '/imgs/module/userGuide.png',
-  name: '用户引导',
+  name: 'core.module.template.User guide',
   intro: userGuideTip,
   inputs: [
     {
       key: ModuleInputKeyEnum.welcomeText,
       type: FlowNodeInputTypeEnum.hidden,
       valueType: ModuleIOValueTypeEnum.string,
-      label: '开场白',
+      label: 'core.app.Welcome Text',
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -23,7 +23,7 @@ export const UserGuideModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.variables,
       type: FlowNodeInputTypeEnum.hidden,
       valueType: ModuleIOValueTypeEnum.any,
-      label: '对话框变量',
+      label: 'core.module.Variable',
       value: [],
       showTargetInApp: false,
       showTargetInPlugin: false
@@ -32,7 +32,7 @@ export const UserGuideModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.questionGuide,
       valueType: ModuleIOValueTypeEnum.boolean,
       type: FlowNodeInputTypeEnum.switch,
-      label: '问题引导',
+      label: '',
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -40,7 +40,7 @@ export const UserGuideModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.tts,
       type: FlowNodeInputTypeEnum.hidden,
       valueType: ModuleIOValueTypeEnum.any,
-      label: '语音播报',
+      label: '',
       showTargetInApp: false,
       showTargetInPlugin: false
     }

packages/global/core/module/template/system/userInput.ts

@@ -15,15 +15,15 @@ export const UserInputModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.questionInput,
   templateType: ModuleTemplateTypeEnum.systemInput,
   flowType: FlowNodeTypeEnum.questionInput,
-  avatar: '/imgs/module/userChatInput.png',
+  avatar: '/imgs/module/userChatInput.svg',
-  name: '用户问题(入口)',
+  name: 'core.module.template.Chat entrance',
-  intro: '用户输入的内容。该模块通常作为应用的入口，用户在发送消息后会首先执行该模块。',
+  intro: 'core.module.template.Chat entrance intro',
   inputs: [
     {
       key: ModuleInputKeyEnum.userChatInput,
       type: FlowNodeInputTypeEnum.systemInput,
       valueType: ModuleIOValueTypeEnum.string,
-      label: '用户问题',
+      label: 'core.module.input.label.user question',
       showTargetInApp: false,
       showTargetInPlugin: false
     }
@@ -31,7 +31,7 @@ export const UserInputModule: FlowModuleTemplateType = {
   outputs: [
     {
       key: ModuleOutputKeyEnum.userChatInput,
-      label: '用户问题',
+      label: 'core.module.input.label.user question',
       type: FlowNodeOutputTypeEnum.source,
       valueType: ModuleIOValueTypeEnum.string,
       targets: []

packages/global/core/module/template/tip.ts

@@ -1,7 +1,4 @@
-export const chatNodeSystemPromptTip =
+export const chatNodeSystemPromptTip = 'core.app.tip.chatNodeSystemPromptTip';
-  '模型固定的引导词，通过调整该内容，可以引导模型聊天方向。该内容会被固定在上下文的开头。可使用变量，例如 {{language}}';
+export const userGuideTip = 'core.app.tip.userGuideTip';
-export const userGuideTip = '可以在对话前设置引导语，设置全局变量，设置下一步指引';
+export const welcomeTextTip = 'core.app.tip.welcomeTextTip';
-export const welcomeTextTip =
+export const variableTip = 'core.app.tip.variableTip';
-  '每次对话开始前，发送一个初始内容。支持标准 Markdown 语法，可使用的额外标记:\n[快捷按键]: 用户点击后可以直接发送该问题';
-export const variableTip =
-  '可以在对话开始前，要求用户填写一些内容作为本轮对话的特定变量。该模块位于开场引导之后。\n变量可以通过 {{变量key}} 的形式注入到其他模块 string 类型的输入中，例如：提示词、限定词等';

packages/global/core/module/type.d.ts

@@ -3,9 +3,9 @@ import { ModuleIOValueTypeEnum, ModuleTemplateTypeEnum, VariableInputEnum } from
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
 
 export type FlowModuleTemplateType = {
-  id: string;
+  id: string; // module id, unique
   templateType: `${ModuleTemplateTypeEnum}`;
-  flowType: `${FlowNodeTypeEnum}`; // unique
+  flowType: `${FlowNodeTypeEnum}`; // render node card
   avatar?: string;
   name: string;
   intro: string; // template list intro
@@ -85,12 +85,14 @@ export type RunningModuleItemType = {
   inputs: {
     key: string;
     value?: any;
+    valueType?: `${ModuleIOValueTypeEnum}`;
   }[];
   outputs: {
     key: string;
     answer?: boolean;
     response?: boolean;
     value?: any;
+    valueType?: `${ModuleIOValueTypeEnum}`;
     targets: {
       moduleId: string;
       key: string;
@@ -115,5 +117,6 @@ export type ChatDispatchProps = {
 
 export type ModuleDispatchProps<T> = ChatDispatchProps & {
   outputs: RunningModuleItemType['outputs'];
-  inputs: T;
+  inputs: RunningModuleItemType['inputs'];
+  params: T;
 };

packages/global/core/module/utils.ts

@@ -1,8 +1,14 @@
 import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from './node/constant';
-import { ModuleIOValueTypeEnum, ModuleInputKeyEnum } from './constants';
+import {
+  ModuleIOValueTypeEnum,
+  ModuleInputKeyEnum,
+  VariableInputEnum,
+  variableMap
+} from './constants';
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
 import { AppTTSConfigType, ModuleItemType, VariableItemType } from './type';
 import { Input_Template_Switch } from './template/input';
+import { EditorVariablePickerType } from '../../../web/components/common/Textarea/PromptEditor/type';
 
 export const getGuideModule = (modules: ModuleItemType[]) =>
   modules.find((item) => item.flowType === FlowNodeTypeEnum.userGuide);
@@ -68,7 +74,7 @@ export function plugin2ModuleIO(
             // plugin id
             key: ModuleInputKeyEnum.pluginId,
             type: FlowNodeInputTypeEnum.hidden,
-            label: 'pluginId',
+            label: '',
             value: pluginId,
             valueType: ModuleIOValueTypeEnum.string,
             connected: true,
@@ -94,3 +100,12 @@ export function plugin2ModuleIO(
       : []
   };
 }
+
+export const formatEditorVariablePickerIcon = (
+  variables: { key: string; label: string; type?: `${VariableInputEnum}` }[]
+): EditorVariablePickerType[] => {
+  return variables.map((item) => ({
+    ...item,
+    icon: item.type ? variableMap[item.type]?.icon : variableMap['input'].icon
+  }));
+};

packages/global/package.json

@@ -7,7 +7,7 @@
     "encoding": "^0.1.13",
     "js-tiktoken": "^1.0.7",
     "openai": "4.23.0",
-    "pdfjs-dist": "^4.0.269",
+    "nanoid": "^4.0.1",
     "timezones-list": "^3.0.2"
   },
   "devDependencies": {

packages/global/support/user/api.d.ts

@@ -3,7 +3,6 @@ import { OAuthEnum } from './constant';
 export type PostLoginProps = {
   username: string;
   password: string;
-  tmbId?: string;
 };
 
 export type OauthLoginProps = {

packages/global/support/user/team/type.d.ts

@@ -9,7 +9,6 @@ export type TeamSchema = {
   createTime: Date;
   balance: number;
   maxSize: number;
-  lastDatasetBillTime: Date;
   limit: {
     lastExportDatasetTime: Date;
     lastWebsiteSyncTime: Date;

packages/global/support/user/type.d.ts

@@ -13,6 +13,7 @@ export type UserModelSchema = {
   createTime: number;
   timezone: string;
   status: `${UserStatusEnum}`;
+  lastLoginTmbId?: string;
   openaiAccount?: {
     key: string;
     baseUrl: string;

packages/global/support/wallet/bill/api.d.ts

@@ -3,8 +3,7 @@ import { BillListItemCountType, BillListItemType } from './type';
 
 export type CreateTrainingBillProps = {
   name: string;
-  vectorModel?: string;
+  datasetId: string;
-  agentModel?: string;
 };
 
 export type ConcatBillProps = BillListItemCountType & {

packages/global/support/wallet/bill/constants.ts

@@ -7,13 +7,28 @@ export enum BillSourceEnum {
   api = 'api',
   shareLink = 'shareLink',
   training = 'training',
-  datasetStore = 'datasetStore'
+
+  standSubPlan = 'standSubPlan',
+  extraDatasetSub = 'extraDatasetSub'
 }
 
-export const BillSourceMap: Record<`${BillSourceEnum}`, string> = {
+export const BillSourceMap = {
-  [BillSourceEnum.fastgpt]: '在线使用',
+  [BillSourceEnum.fastgpt]: {
-  [BillSourceEnum.api]: 'Api',
+    label: '在线使用'
-  [BillSourceEnum.shareLink]: '免登录链接',
+  },
-  [BillSourceEnum.training]: '数据训练',
+  [BillSourceEnum.api]: {
-  [BillSourceEnum.datasetStore]: '知识库存储'
+    label: 'Api'
+  },
+  [BillSourceEnum.shareLink]: {
+    label: '免登录链接'
+  },
+  [BillSourceEnum.training]: {
+    label: 'dataset.Training Name'
+  },
+  [BillSourceEnum.standSubPlan]: {
+    label: 'support.wallet.subscription.type.standard'
+  },
+  [BillSourceEnum.extraDatasetSub]: {
+    label: 'support.wallet.subscription.type.extraDatasetSize'
+  }
 };

packages/global/support/wallet/bill/type.d.ts

@@ -4,9 +4,11 @@ import { BillSourceEnum } from './constants';
 export type BillListItemCountType = {
   inputTokens?: number;
   outputTokens?: number;
-  textLen?: number;
+  charsLength?: number;
   duration?: number;
-  dataLen?: number;
+
+  // sub
+  datasetSize?: number;
 
   // abandon
   tokenLen?: number;
@@ -24,7 +26,7 @@ export type BillSchema = CreateBillProps & {
 
 export type BillItemType = {
   id: string;
-  memberName: string;
+  // memberName: string;
   time: Date;
   appName: string;
   source: BillSchema['source'];

packages/global/support/wallet/pay/constants.ts

@@ -0,0 +1,41 @@
+export enum PayTypeEnum {
+  balance = 'balance',
+  subStandard = 'subStandard',
+  subExtraDatasetSize = 'subExtraDatasetSize',
+  subExtraPoints = 'subExtraPoints'
+}
+export const payTypeMap = {
+  [PayTypeEnum.balance]: {
+    label: 'support.user.team.pay.type.balance'
+  },
+  [PayTypeEnum.subStandard]: {
+    label: 'support.wallet.subscription.type.standard'
+  },
+  [PayTypeEnum.subExtraDatasetSize]: {
+    label: 'support.wallet.subscription.type.extraDatasetSize'
+  },
+  [PayTypeEnum.subExtraPoints]: {
+    label: 'support.wallet.subscription.type.extraPoints'
+  }
+};
+
+export enum PayStatusEnum {
+  SUCCESS = 'SUCCESS',
+  REFUND = 'REFUND',
+  NOTPAY = 'NOTPAY',
+  CLOSED = 'CLOSED'
+}
+export const payStatusMap = {
+  [PayStatusEnum.SUCCESS]: {
+    label: 'support.user.team.pay.status.success'
+  },
+  [PayStatusEnum.REFUND]: {
+    label: 'support.user.team.pay.status.refund'
+  },
+  [PayStatusEnum.NOTPAY]: {
+    label: 'support.user.team.pay.status.notpay'
+  },
+  [PayStatusEnum.CLOSED]: {
+    label: 'support.user.team.pay.status.closed'
+  }
+};

packages/global/support/wallet/pay/type.d.ts

@@ -1,10 +1,18 @@
+import { SubModeEnum, SubTypeEnum } from '../sub/constants';
+import { PayTypeEnum } from './constants';
+
 export type PaySchema = {
   _id: string;
   userId: string;
   teamId: string;
   tmbId: string;
   createTime: Date;
-  price: number;
   orderId: string;
   status: 'SUCCESS' | 'REFUND' | 'NOTPAY' | 'CLOSED';
+  type: `${PayType}`;
+
+  price: number;
+  payWay: 'balance' | 'wx';
+
+  subMetadata: {};
 };

packages/global/support/wallet/sub/api.d.ts

@@ -0,0 +1,36 @@
+import { StandardSubLevelEnum, SubModeEnum } from './constants';
+import { TeamSubSchema } from './type.d';
+
+export type SubDatasetSizeParams = {
+  size: number;
+};
+export type StandardSubPlanParams = {
+  level: `${StandardSubLevelEnum}`;
+  mode: `${SubModeEnum}`;
+};
+
+export type SubDatasetSizePreviewCheckResponse = {
+  payForNewSub: boolean; // Does this change require payment
+  newSubSize: number; // new sub dataset size
+  alreadySubSize: number; // old sub dataset size
+  payPrice: number; // this change require payment
+  newPlanPrice: number; // the new sub price
+  newSubStartTime: Date;
+  newSubExpiredTime: Date;
+  balanceEnough: boolean; // team balance is enough
+};
+export type StandardSubPlanUpdateResponse = {
+  balanceEnough: boolean; // team balance is enough
+  payPrice?: number;
+  planPrice: number;
+  planPointPrice: number;
+
+  currentMode: `${SubModeEnum}`;
+  nextMode: `${SubModeEnum}`;
+  currentSubLevel: `${StandardSubLevelEnum}`;
+  nextSubLevel: `${StandardSubLevelEnum}`;
+  totalPoints: number;
+  surplusPoints: number;
+  planStartTime: Date;
+  planExpiredTime: Date;
+};

packages/global/support/wallet/sub/constants.ts

@@ -0,0 +1,80 @@
+export const POINTS_SCALE = 10000;
+
+export enum SubTypeEnum {
+  standard = 'standard',
+  extraDatasetSize = 'extraDatasetSize',
+  extraPoints = 'extraPoints'
+}
+export const subTypeMap = {
+  [SubTypeEnum.standard]: {
+    label: 'support.wallet.subscription.type.standard'
+  },
+  [SubTypeEnum.extraDatasetSize]: {
+    label: 'support.wallet.subscription.type.extraDatasetSize'
+  },
+  [SubTypeEnum.extraPoints]: {
+    label: 'support.wallet.subscription.type.extraPoints'
+  }
+};
+
+export enum SubStatusEnum {
+  active = 'active',
+  canceled = 'canceled'
+}
+export const subStatusMap = {
+  [SubStatusEnum.active]: {
+    label: 'support.wallet.subscription.status.active'
+  },
+  [SubStatusEnum.canceled]: {
+    label: 'support.wallet.subscription.status.canceled'
+  }
+};
+export const subSelectMap = {
+  true: SubStatusEnum.active,
+  false: SubStatusEnum.canceled
+};
+
+export enum SubModeEnum {
+  month = 'month',
+  year = 'year'
+}
+export const subModeMap = {
+  [SubModeEnum.month]: {
+    label: 'support.wallet.subscription.mode.month',
+    durationMonth: 1
+  },
+  [SubModeEnum.year]: {
+    label: 'support.wallet.subscription.mode.year',
+    durationMonth: 12
+  }
+};
+
+export enum StandardSubLevelEnum {
+  free = 'free',
+  experience = 'experience',
+  team = 'team',
+  enterprise = 'enterprise',
+  custom = 'custom'
+}
+export const standardSubLevelMap = {
+  [StandardSubLevelEnum.free]: {
+    label: 'support.wallet.subscription.standardSubLevel.free',
+    desc: 'support.wallet.subscription.standardSubLevel.free desc'
+  },
+  [StandardSubLevelEnum.experience]: {
+    label: 'support.wallet.subscription.standardSubLevel.experience',
+    desc: 'support.wallet.subscription.standardSubLevel.experience desc'
+  },
+  [StandardSubLevelEnum.team]: {
+    label: 'support.wallet.subscription.standardSubLevel.team',
+    desc: ''
+  },
+  [StandardSubLevelEnum.enterprise]: {
+    label: 'support.wallet.subscription.standardSubLevel.enterprise',
+    desc: ''
+  },
+  [StandardSubLevelEnum.custom]: {
+    label: 'support.wallet.subscription.standardSubLevel.custom',
+    desc: ''
+  }
+};

packages/global/support/wallet/sub/type.d.ts

@@ -0,0 +1,73 @@
+import { StandardSubLevelEnum, SubModeEnum, SubStatusEnum, SubTypeEnum } from './constants';
+
+// Content of plan
+export type TeamStandardSubPlanItemType = {
+  price: number; // read price
+  pointPrice: number; // read price/ one ten thousand
+  maxTeamMember: number;
+  maxAppAmount: number; // max app or plugin amount
+  maxDatasetAmount: number;
+  chatHistoryStoreDuration: number; // n day
+  maxDatasetSize: number;
+  customApiKey: boolean;
+  customCopyright: boolean; // feature
+  websiteSyncInterval: number; // n hours
+  trainingWeight: number; // 1~4
+  reRankWeight: number; // 1~4
+  totalPoints: number; // n ten thousand
+};
+
+export type StandSubPlanLevelMapType = Record<
+  `${StandardSubLevelEnum}`,
+  TeamStandardSubPlanItemType
+>;
+
+export type SubPlanType = {
+  [SubTypeEnum.standard]: StandSubPlanLevelMapType;
+  [SubTypeEnum.extraDatasetSize]: {
+    price: number;
+  };
+};
+
+export type TeamSubSchema = {
+  _id: string;
+  teamId: string;
+  type: `${SubTypeEnum}`;
+  status: `${SubStatusEnum}`;
+  currentMode: `${SubModeEnum}`;
+  nextMode: `${SubModeEnum}`;
+  startTime: Date;
+  expiredTime: Date;
+  price: number;
+
+  currentSubLevel: `${StandardSubLevelEnum}`;
+  nextSubLevel: `${StandardSubLevelEnum}`;
+  pointPrice: number;
+  totalPoints: number;
+
+  currentExtraDatasetSize: number;
+  nextExtraDatasetSize: number;
+
+  currentExtraPoints: number;
+  nextExtraPoints: number;
+
+  surplusPoints: number;
+
+  // abandon
+  datasetStoreAmount?: number;
+  renew?: boolean;
+};
+
+export type FeTeamSubType = {
+  [SubTypeEnum.standard]?: TeamSubSchema;
+  [SubTypeEnum.extraDatasetSize]?: TeamSubSchema;
+  [SubTypeEnum.extraPoints]?: TeamSubSchema;
+
+  standardMaxDatasetSize: number;
+  totalPoints: number;
+  usedPoints: number;
+
+  standardMaxPoints: number;
+  datasetMaxSize: number;
+  usedDatasetSize: number;
+};

packages/service/common/file/constants.ts

@@ -0,0 +1,6 @@
+import path from 'path';
+
+export const tmpFileDirPath =
+  process.env.NODE_ENV === 'production' ? '/app/tmp' : path.join(process.cwd(), 'tmp');
+
+export const previewMaxCharCount = 3000;

packages/service/common/file/gridfs/controller.ts

@@ -3,9 +3,10 @@ import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import fsp from 'fs/promises';
 import fs from 'fs';
 import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
-import { delImgByFileIdList } from '../image/controller';
+import { MongoFileSchema } from './schema';
 
 export function getGFSCollection(bucket: `${BucketNameEnum}`) {
+  MongoFileSchema;
   return connectionMongo.connection.db.collection(`${bucket}.files`);
 }
 export function getGridBucket(bucket: `${BucketNameEnum}`) {
@@ -21,6 +22,7 @@ export async function uploadFile({
   tmbId,
   path,
   filename,
+  contentType,
   metadata = {}
 }: {
   bucketName: `${BucketNameEnum}`;
@@ -28,6 +30,7 @@ export async function uploadFile({
   tmbId: string;
   path: string;
   filename: string;
+  contentType?: string;
   metadata?: Record<string, any>;
 }) {
   if (!path) return Promise.reject(`filePath is empty`);
@@ -44,7 +47,7 @@ export async function uploadFile({
 
   const stream = bucket.openUploadStream(filename, {
     metadata,
-    contentType: metadata?.contentType
+    contentType
   });
 
   // save to gridfs
@@ -96,40 +99,6 @@ export async function delFileByFileIdList({
     }
   }
 }
-// delete file by metadata(datasetId)
-export async function delFileByMetadata({
-  bucketName,
-  datasetId
-}: {
-  bucketName: `${BucketNameEnum}`;
-  datasetId?: string;
-}) {
-  const bucket = getGridBucket(bucketName);
-
-  const files = await bucket
-    .find(
-      {
-        ...(datasetId && { 'metadata.datasetId': datasetId })
-      },
-      {
-        projection: {
-          _id: 1
-        }
-      }
-    )
-    .toArray();
-
-  const idList = files.map((item) => String(item._id));
-
-  // delete img
-  await delImgByFileIdList(idList);
-
-  // delete file
-  await delFileByFileIdList({
-    bucketName,
-    fileIdList: idList
-  });
-}
 
 export async function getDownloadStream({
   bucketName,

packages/service/common/file/gridfs/schema.ts

@@ -0,0 +1,15 @@
+import { connectionMongo, type Model } from '../../mongo';
+const { Schema, model, models } = connectionMongo;
+
+const FileSchema = new Schema({});
+
+try {
+  FileSchema.index({ 'metadata.teamId': 1 });
+  FileSchema.index({ 'metadata.uploadDate': -1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoFileSchema = models['dataset.files'] || model('dataset.files', FileSchema);
+
+MongoFileSchema.syncIndexes();

packages/service/common/file/image/constant.ts

@@ -1 +0,0 @@
-export const imageBaseUrl = '/api/system/img/';

packages/service/common/file/image/controller.ts

@@ -1,6 +1,7 @@
 import { UploadImgProps } from '@fastgpt/global/common/file/api';
-import { imageBaseUrl } from './constant';
+import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants';
 import { MongoImage } from './schema';
+import { ClientSession } from '../../../common/mongo';
 
 export function getMongoImgUrl(id: string) {
   return `${imageBaseUrl}${id}`;
@@ -8,10 +9,13 @@ export function getMongoImgUrl(id: string) {
 
 export const maxImgSize = 1024 * 1024 * 12;
 export async function uploadMongoImg({
+  type,
   base64Img,
   teamId,
   expiredTime,
-  metadata
+  metadata,
+
+  shareId
 }: UploadImgProps & {
   teamId: string;
 }) {
@@ -20,12 +24,16 @@ export async function uploadMongoImg({
   }
 
   const base64Data = base64Img.split(',')[1];
+  const binary = Buffer.from(base64Data, 'base64');
 
   const { _id } = await MongoImage.create({
+    type,
     teamId,
-    binary: Buffer.from(base64Data, 'base64'),
+    binary,
     expiredTime: expiredTime,
-    metadata
+    metadata,
+
+    shareId
   });
 
   return getMongoImgUrl(String(_id));
@@ -39,8 +47,22 @@ export async function readMongoImg({ id }: { id: string }) {
   return data?.binary;
 }
 
-export async function delImgByFileIdList(fileIds: string[]) {
+export async function delImgByRelatedId({
-  return MongoImage.deleteMany({
+  teamId,
-    'metadata.fileId': { $in: fileIds.map((item) => String(item)) }
+  relateIds,
-  });
+  session
+}: {
+  teamId: string;
+  relateIds: string[];
+  session: ClientSession;
+}) {
+  if (relateIds.length === 0) return;
+
+  return MongoImage.deleteMany(
+    {
+      teamId,
+      'metadata.relatedId': { $in: relateIds.map((id) => String(id)) }
+    },
+    { session }
+  );
 }

packages/service/common/file/image/schema.ts

@@ -1,5 +1,7 @@
 import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
 import { connectionMongo, type Model } from '../../mongo';
+import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d';
+import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants';
 const { Schema, model, models } = connectionMongo;
 
 const ImageSchema = new Schema({
@@ -12,12 +14,18 @@ const ImageSchema = new Schema({
     type: Date,
     default: () => new Date()
   },
-  binary: {
-    type: Buffer
-  },
   expiredTime: {
     type: Date
   },
+  binary: {
+    type: Buffer
+  },
+  type: {
+    type: String,
+    enum: Object.keys(mongoImageTypeMap),
+    required: true
+  },
+
   metadata: {
     type: Object
   }
@@ -25,14 +33,14 @@ const ImageSchema = new Schema({
 
 try {
   ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
+  ImageSchema.index({ type: 1 });
+  ImageSchema.index({ createTime: 1 });
+  ImageSchema.index({ teamId: 1, 'metadata.relatedId': 1 });
 } catch (error) {
   console.log(error);
 }
 
-export const MongoImage: Model<{
+export const MongoImage: Model<MongoImageSchemaType> =
-  teamId: string;
+  models['image'] || model('image', ImageSchema);
-  binary: Buffer;
-  metadata?: { fileId?: string };
-}> = models['image'] || model('image', ImageSchema);
 
 MongoImage.syncIndexes();

packages/service/common/file/multer.ts

@@ -1,11 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
-import { customAlphabet } from 'nanoid';
 import multer from 'multer';
 import path from 'path';
 import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants';
-import fs from 'fs';
+import { getNanoid } from '@fastgpt/global/common/string/tools';
-
-const nanoid = customAlphabet('1234567890abcdef', 12);
 
 type FileType = {
   fieldname: string;
@@ -17,7 +14,7 @@ type FileType = {
   size: number;
 };
 
-export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) {
+export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
   maxSize *= 1024 * 1024;
   class UploadModel {
     uploader = multer({
@@ -26,17 +23,25 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) {
       },
       preservePath: true,
       storage: multer.diskStorage({
-        filename: (_req, file, cb) => {
+        // destination: (_req, _file, cb) => {
+        //   cb(null, tmpFileDirPath);
+        // },
+        filename: async (req, file, cb) => {
           const { ext } = path.parse(decodeURIComponent(file.originalname));
-          cb(null, nanoid() + ext);
+          cb(null, `${getNanoid()}${ext}`);
         }
       })
-    }).any();
+    }).single('file');
 
-    async doUpload<T = Record<string, any>>(req: NextApiRequest, res: NextApiResponse) {
+    async doUpload<T = Record<string, any>>(
+      req: NextApiRequest,
+      res: NextApiResponse,
+      originBuckerName?: `${BucketNameEnum}`
+    ) {
       return new Promise<{
-        files: FileType[];
+        file: FileType;
-        metadata: T;
+        metadata: Record<string, any>;
+        data: T;
         bucketName?: `${BucketNameEnum}`;
       }>((resolve, reject) => {
         // @ts-ignore
@@ -46,25 +51,33 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) {
           }
 
           // check bucket name
-          const bucketName = req.body?.bucketName as `${BucketNameEnum}`;
+          const bucketName = (req.body?.bucketName || originBuckerName) as `${BucketNameEnum}`;
           if (bucketName && !bucketNameMap[bucketName]) {
             return reject('BucketName is invalid');
           }
 
+          // @ts-ignore
+          const file = req.file as FileType;
+
           resolve({
-            ...req.body,
+            file: {
-            files:
+              ...file,
-              // @ts-ignore
+              originalname: decodeURIComponent(file.originalname)
-              req.files?.map((file) => ({
+            },
-                ...file,
+            bucketName,
-                originalname: decodeURIComponent(file.originalname)
-              })) || [],
             metadata: (() => {
               if (!req.body?.metadata) return {};
               try {
                 return JSON.parse(req.body.metadata);
               } catch (error) {
-                console.log(error);
+                return {};
+              }
+            })(),
+            data: (() => {
+              if (!req.body?.data) return {};
+              try {
+                return JSON.parse(req.body.data);
+              } catch (error) {
                 return {};
               }
             })()
@@ -75,14 +88,4 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) {
   }
 
   return new UploadModel();
-}
-
-export const removeFilesByPaths = (paths: string[]) => {
-  paths.forEach((path) => {
-    fs.unlink(path, (err) => {
-      if (err) {
-        console.error(err);
-      }
-    });
-  });
 };

packages/service/common/file/utils.ts

@@ -0,0 +1,11 @@
+import fs from 'fs';
+
+export const removeFilesByPaths = (paths: string[]) => {
+  paths.forEach((path) => {
+    fs.unlink(path, (err) => {
+      if (err) {
+        // console.error(err);
+      }
+    });
+  });
+};

packages/service/common/mongo/sessionRun.ts

@@ -1,21 +1,21 @@
-import mongoose, { connectionMongo } from './index';
+import { connectionMongo, ClientSession } from './index';
 
-export async function mongoSessionTask(
+export const mongoSessionRun = async <T = unknown>(fn: (session: ClientSession) => Promise<T>) => {
-  fn: (session: mongoose.mongo.ClientSession) => Promise<any>
-) {
   const session = await connectionMongo.startSession();
+  session.startTransaction();
 
   try {
-    session.startTransaction();
+    const result = await fn(session);
-
-    await fn(session);
 
     await session.commitTransaction();
-    await session.endSession();
+    session.endSession();
+
+    return result as T;
   } catch (error) {
+    console.log(error);
+
     await session.abortTransaction();
-    await session.endSession();
+    session.endSession();
-    console.error(error);
     return Promise.reject(error);
   }
-}
+};

packages/service/common/response/index.ts

@@ -3,6 +3,7 @@ import { sseResponseEventEnum } from './constant';
 import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
 import { addLog } from '../system/log';
 import { clearCookie } from '../../support/permission/controller';
+import { replaceSensitiveLink } from '@fastgpt/global/common/string/tools';
 
 export interface ResponseType<T = any> {
   code: number;
@@ -52,7 +53,7 @@ export const jsonRes = <T = any>(
   res.status(code).json({
     code,
     statusText: '',
-    message: message || msg,
+    message: replaceSensitiveLink(message || msg),
     data: data !== undefined ? data : null
   });
 };
@@ -90,7 +91,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => {
   responseWrite({
     res,
     event: sseResponseEventEnum.error,
-    data: JSON.stringify({ message: msg })
+    data: JSON.stringify({ message: replaceSensitiveLink(msg) })
   });
 };
 

packages/service/common/string/cheerio.ts

@@ -50,8 +50,11 @@ export const cheerioToHtml = ({
     .get()
     .join('\n');
 
+  const title = $('head title').text() || $('h1:first').text() || fetchUrl;
+
   return {
     html,
+    title,
     usedSelector
   };
 };
@@ -61,39 +64,39 @@ export const urlsFetch = async ({
 }: UrlFetchParams): Promise<UrlFetchResponse> => {
   urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
 
-  const response = (
+  const response = await Promise.all(
-    await Promise.all(
+    urlList.map(async (url) => {
-      urlList.map(async (url) => {
+      try {
-        try {
+        const fetchRes = await axios.get(url, {
-          const fetchRes = await axios.get(url, {
+          timeout: 30000
-            timeout: 30000
+        });
-          });
 
-          const $ = cheerio.load(fetchRes.data);
+        const $ = cheerio.load(fetchRes.data);
-          const { html, usedSelector } = cheerioToHtml({
+        const { title, html, usedSelector } = cheerioToHtml({
-            fetchUrl: url,
+          fetchUrl: url,
-            $,
+          $,
-            selector
+          selector
-          });
+        });
-          const md = await htmlToMarkdown(html);
+        const md = await htmlToMarkdown(html);
 
-          return {
+        return {
-            url,
+          url,
-            content: md,
+          title,
-            selector: usedSelector
+          content: md,
-          };
+          selector: usedSelector
-        } catch (error) {
+        };
-          console.log(error, 'fetch error');
+      } catch (error) {
+        console.log(error, 'fetch error');
 
-          return {
+        return {
-            url,
+          url,
-            content: '',
+          title: '',
-            selector: ''
+          content: '',
-          };
+          selector: ''
-        }
+        };
-      })
+      }
-    )
+    })
-  ).filter((item) => item.content);
+  );
 
   return response;
 };

packages/service/common/string/markdown.ts

@@ -15,7 +15,9 @@ export const htmlToMarkdown = (html?: string | null) =>
     worker.on('message', (md: string) => {
       worker.terminate();
 
-      resolve(simpleMarkdownText(md));
+      let rawText = simpleMarkdownText(md);
+
+      resolve(rawText);
     });
     worker.on('error', (err) => {
       worker.terminate();

packages/service/common/system/cron.ts

@@ -0,0 +1,6 @@
+import nodeCron from 'node-cron';
+
+export const setCron = (time: string, cb: () => void) => {
+  // second minute hour day month week
+  return nodeCron.schedule(time, cb);
+};