fix: CQ module output (#445)

文档改为https://doc.fastgpt.in网站访问

.github/workflows/preview-image.yml

@@ -15,7 +15,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
         with:
-          fetch-depth: 1
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          submodules: recursive # Fetch submodules
+          fetch-depth: 0 # Fetch all history for .GitInfo and .Lastmod
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
         with:

Dockerfile

@@ -1,5 +1,5 @@
 # Install dependencies only when needed
-FROM node:current-alpine AS deps
+FROM node:18.15-alpine AS deps
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk add --no-cache libc6-compat && npm install -g pnpm
 WORKDIR /app
@@ -11,12 +11,12 @@ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY ./packages ./packages
 COPY ./projects/$name/package.json ./projects/$name/package.json
 
-RUN \
-  [ -f pnpm-lock.yaml ] && pnpm install || \
-  (echo "Lockfile not found." && exit 1)
+RUN [ -f pnpm-lock.yaml ] || (echo "Lockfile not found." && exit 1)
+
+RUN pnpm install
 
 # Rebuild the source code only when needed
-FROM node:current-alpine AS builder
+FROM node:18.15-alpine AS builder
 WORKDIR /app
 
 ARG name
@@ -33,7 +33,7 @@ ENV NEXT_TELEMETRY_DISABLED 1
 RUN npm install -g pnpm
 RUN pnpm --filter=$name run build
 
-FROM node:current-alpine AS runner
+FROM node:18.15-alpine AS runner
 WORKDIR /app
 
 ARG name

README.md

@@ -15,19 +15,19 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
 
 <p align="center">
   <a href="https://fastgpt.run/">
-    <img height="21" src="https://img.shields.io/badge/在线使用-fff?style=flat-square&logo=spoj&logoColor=7d09f1" alt="cloud">
+    <img height="21" src="https://img.shields.io/badge/在线使用-d4eaf7?style=flat-square&logo=spoj&logoColor=7d09f1" alt="cloud">
   </a>
   <a href="https://doc.fastgpt.run/docs/intro">
     <img height="21" src="https://img.shields.io/badge/相关文档-7d09f1?style=flat-square" alt="document">
   </a>
   <a href="https://doc.fastgpt.run/docs/development">
-    <img height="21" src="https://img.shields.io/badge/本地开发-%23fff?style=flat-square&logo=xcode&logoColor=7d09f1" alt="development">
+    <img height="21" src="https://img.shields.io/badge/本地开发-%23d4eaf7?style=flat-square&logo=xcode&logoColor=7d09f1" alt="development">
   </a>
   <a href="/#-%E7%9B%B8%E5%85%B3%E9%A1%B9%E7%9B%AE">
     <img height="21" src="https://img.shields.io/badge/相关项目-7d09f1?style=flat-square" alt="project">
   </a>
   <a href="https://github.com/labring/FastGPT/blob/main/LICENSE">
-    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=fff&color=7d09f1" alt="license">
+    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=7d09f1" alt="license">
   </a>
 </p>
 
@@ -35,7 +35,8 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 ## 🛸 在线使用
 
-[fastgpt.run](https://fastgpt.run/)（服务器在新加坡，部分地区可能无法直连）
+- 🌐 国内版：[ai.fastgpt.in](https://ai.fastgpt.in/)
+- 🌍 海外版：[fastgpt.run](https://fastgpt.run/)
 
 |                                    |                                    |
 | ---------------------------------- | ---------------------------------- |
@@ -46,7 +47,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 1. 强大的可视化编排，轻松构建 AI 应用
    - [x] 提供简易模式，无需操作编排
-   - [x] 用户对话前引导, 全局字符串变量
+   - [x] 用户对话前引导，全局字符串变量
    - [x] 知识库搜索
    - [x] 多 LLM 模型对话
    - [x] 文本内容提取成结构化数据
@@ -55,12 +56,12 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
    - [x] 对话下一步指引
    - [ ] 对话多路线选择
    - [x] 源文件引用追踪
-   - [ ] 自定义文件阅读器
+   - [x] 模块封装，实现多级复用
 2. 丰富的知识库预处理
    - [x] 多库复用，混用
    - [x] chunk 记录修改和删除
-   - [x] 支持 手动输入, 直接分段, QA 拆分导入
-   - [x] 支持 url 读取、 CSV 批量导入
+   - [x] 支持手动输入，直接分段，QA 拆分导入
+   - [x] 支持 url 读取、CSV 批量导入
    - [x] 支持知识库单独设置向量模型
    - [x] 源文件存储
    - [ ] 文件学习 Agent
@@ -70,7 +71,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
    - [x] 完整上下文呈现
    - [x] 完整模块中间值呈现
 4. OpenAPI
-   - [x] completions 接口（对齐 GPT 接口）
+   - [x] completions 接口 (对齐 GPT 接口)
    - [ ] 知识库 CRUD
 5. 运营功能
    - [x] 免登录分享窗口
@@ -79,7 +80,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 ## 👨‍💻 开发
 
-项目技术栈: NextJs + TS + ChakraUI + Mongo + Postgres（Vector 插件）
+项目技术栈：NextJs + TS + ChakraUI + Mongo + Postgres (Vector 插件)
 
 - **⚡ 快速部署**
 
@@ -89,12 +90,13 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
   由于需要部署数据库，部署完后需要等待 2~4 分钟才能正常访问。默认用了最低配置，首次访问时会有些慢。
 
-* [快开始本地开发](https://doc.fastgpt.run/docs/development/intro/)
-* [部署 FastGPT](https://doc.fastgpt.run/docs/installation)
-* [系统配置文件说明](https://doc.fastgpt.run/docs/development/configuration/)
-* [多模型配置](https://doc.fastgpt.run/docs/installation/one-api/)
-* [版本更新/升级介绍](https://doc.fastgpt.run/docs/installation/upgrading)
-* [API 文档](https://doc.fastgpt.run/docs/development/openapi/)
+* [快开始本地开发](https://doc.fastgpt.in/docs/development/intro/)
+* [部署 FastGPT](https://doc.fastgpt.in/docs/installation)
+* [系统配置文件说明](https://doc.fastgpt.in/docs/development/configuration/)
+* [多模型配置](https://doc.fastgpt.in/docs/installation/one-api/)
+* [版本更新/升级介绍](https://doc.fastgpt.in/docs/installation/upgrading)
+* [OpenAPI API 文档](https://doc.fastgpt.in/docs/development/openapi/)
+* [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
 
 ## 🏘️ 社区交流群
 
@@ -104,10 +106,10 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 ## 💪 相关项目
 
-- [Laf: 3 分钟快速接入三方应用](https://github.com/labring/laf)
-- [Sealos: 快速部署集群应用](https://github.com/labring/sealos)
-- [One API: 多模型管理，支持 Azure、文心一言等](https://github.com/songquanpeng/one-api)
-- [TuShan: 5 分钟搭建后台管理系统](https://github.com/msgbyte/tushan)
+- [Laf：3 分钟快速接入三方应用](https://github.com/labring/laf)
+- [Sealos：快速部署集群应用](https://github.com/labring/sealos)
+- [One API：多模型管理，支持 Azure、文心一言等](https://github.com/songquanpeng/one-api)
+- [TuShan：5 分钟搭建后台管理系统](https://github.com/msgbyte/tushan)
 
 ## 👀 其他
 
@@ -127,7 +129,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 本仓库遵循 [FastGPT Open Source License](./LICENSE) 开源协议。
 
-1. 允许作为后台服务直接商用，但不允许直接使用 SaaS 服务商用。
-2. 需保留相关版权信息。
+1. 允许作为后台服务直接商用，但不允许提供 SaaS 服务。
+2. 未经商业授权，任何形式的商用服务均需保留相关版权信息。
 3. 完整请查看 [FastGPT Open Source License](./LICENSE)
-4. 联系方式：yujinlong@sealos.io, [点击查看定价策略](https://doc.fastgpt.run/docs/commercial)
+4. 联系方式：yujinlong@sealos.io，[点击查看商业版定价策略](https://doc.fastgpt.run/docs/commercial)

README_en.md

@@ -15,19 +15,19 @@ FastGPT is a knowledge-based Q&A system built on the LLM, offers out-of-the-box
 
 <p align="center">
   <a href="https://fastgpt.run/">
-    <img height="21" src="https://img.shields.io/badge/Website-fff?style=flat-square&logo=spoj&logoColor=7d09f1" alt="cloud">
+    <img height="21" src="https://img.shields.io/badge/在线使用-d4eaf7?style=flat-square&logo=spoj&logoColor=7d09f1" alt="cloud">
   </a>
   <a href="https://doc.fastgpt.run/docs/intro">
-    <img height="21" src="https://img.shields.io/badge/Docs-7d09f1?style=flat-square" alt="document">
+    <img height="21" src="https://img.shields.io/badge/相关文档-7d09f1?style=flat-square" alt="document">
   </a>
   <a href="https://doc.fastgpt.run/docs/development">
-    <img height="21" src="https://img.shields.io/badge/Development-%23fff?style=flat-square&logo=xcode&logoColor=7d09f1" alt="development">
+    <img height="21" src="https://img.shields.io/badge/本地开发-%23d4eaf7?style=flat-square&logo=xcode&logoColor=7d09f1" alt="development">
   </a>
   <a href="/#-%E7%9B%B8%E5%85%B3%E9%A1%B9%E7%9B%AE">
-    <img height="21" src="https://img.shields.io/badge/Related Projects-7d09f1?style=flat-square" alt="project">
+    <img height="21" src="https://img.shields.io/badge/相关项目-7d09f1?style=flat-square" alt="project">
   </a>
   <a href="https://github.com/labring/FastGPT/blob/main/LICENSE">
-    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=fff&color=7d09f1" alt="license">
+    <img height="21" src="https://img.shields.io/badge/License-Apache--2.0-ffffff?style=flat-square&labelColor=d4eaf7&color=7d09f1" alt="license">
   </a>
 </p>
 
@@ -54,9 +54,9 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
    - [x] Extend with HTTP
    - [ ] Embed Laf for on-the-fly HTTP module crafting
    - [x] Directions for the next dialogue steps
-   - [ ] Multiple dialogue paths selection
    - [x] Tracking source file references
    - [ ] Custom file reader
+   - [ ] Modules are packaged into plug-ins to achieve reuse
 
 2. Extensive knowledge base preprocessing
 

docSite/.zhlintignore

@@ -0,0 +1 @@
+*.html

docSite/.zhlintrc

@@ -0,0 +1,6 @@
+{
+  "preset": "default",
+  "rules": {
+    "adjustedFullWidthPunctuation": ""
+  }
+}

docSite/README.md

@@ -3,7 +3,7 @@
 ## 本地运行
 
 1. 安装 go 语言环境。
-2. 安装 hugo。 [二进制下载](https://github.com/gohugoio/hugo/releases/tag/v0.117.0)，注意需要安装 extended 版本。
+2. 安装 hugo。[二进制下载](https://github.com/gohugoio/hugo/releases/tag/v0.117.0)，注意需要安装 extended 版本。
 3. cd docSite
 4. hugo serve
 5. 访问 http://localhost:1313

docSite/content/docs/development/configuration.md

@@ -23,7 +23,7 @@ weight: 520
   "SystemParams": {
     "vectorMaxProcess": 15, // 向量生成最大进程，结合数据库性能和 key 来设置
     "qaMaxProcess": 15,  // QA 生成最大进程，结合数据库性能和 key 来设置
-    "pgHNSWEfSearch": 40  // pg vector 索引参数，越大精度高但速度慢
+    "pgHNSWEfSearch": 100  // pg vector 索引参数，越大精度高但速度慢
   },
   "ChatModels": [
     {
@@ -54,7 +54,7 @@ weight: 520
       "defaultSystemChatPrompt": ""
     }
   ],
-  "QAModel": [ // QA 拆分模型
+  "QAModels": [ // QA 拆分模型
     { 
       "model": "gpt-3.5-turbo-16k",
       "name": "GPT35-16k",

docSite/content/docs/development/intro.md

@@ -54,7 +54,7 @@ git clone git@github.com:<github_username>/FastGPT.git
 
 **环境变量**
 
-复制.env.template 文件，生成一个.env.local 环境变量文件夹，修改.env.local 里内容才是有效的变量。变量说明见 .env.template
+复制.env.template 文件，在同级目录下生成一个.env.local 文件，修改.env.local 里内容才是有效的变量。变量说明见 .env.template
 
 **config 配置文件**
 

docSite/content/docs/development/openApi.md

@@ -120,7 +120,7 @@ event: answer
 data: [DONE]
 
 event: appStreamResponse
-data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2","tokens":6,"similarity":0.61,"limit":3},{"moduleName":"AI Chat","price":463.5,"model":"FastAI-4k","tokens":309,"question":"导演是谁","answer":"电影《铃芽之旅》的导演是新海诚。","maxToken":2050,"quoteList":[{"kb_id":"646627f4f7b896cfd8910e38","id":"8099","q":"本作的主人公是谁？","a":"本作的主人公是名叫铃芽的少女。","source":"手动修改"},{"kb_id":"646627f4f7b896cfd8910e38","id":"8686","q":"电影《铃芽之旅》男主角是谁？","a":"电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。","source":""},{"kb_id":"646627f4f7b896cfd8910e38","id":"19339","q":"电影《铃芽之旅》的导演是谁？22","a":"电影《铃芽之旅》的导演是新海诚。","source":"手动修改"}],"completeMessages":[{"obj":"System","value":"下面是知识库内容:\n1. [本作的主人公是谁？\n本作的主人公是名叫铃芽的少女。]\n2. [电影《铃芽之旅》男主角是谁？\n电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。]\n3. [电影《铃芽之旅》的导演是谁？22\n电影《铃芽之旅》的导演是新海诚。]\n"},{"obj":"System","value":"1.请记住，你的身份是百度的下一代知识增强语言模型，能够完全根据知识库提供的内容回答问题。\n\n2. 你忘记了关于电影《铃芽之旅》以外的内容。"},{"obj":"System","value":"你仅回答关于电影《玲芽之旅》的问题，其余问题直接回复: 我不清楚。"},{"obj":"Human","value":"导演是谁"},{"obj":"AI","value":"电影《铃芽之旅》的导演是新海诚。"}]}]
+data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2","tokens":6,"similarity":0.61,"limit":3},{"moduleName":"AI Chat","price":463.5,"model":"FastAI-4k","tokens":309,"question":"导演是谁","answer":"电影《铃芽之旅》的导演是新海诚。","maxToken":2050,"quoteList":[{"dataset_id":"646627f4f7b896cfd8910e38","id":"8099","q":"本作的主人公是谁？","a":"本作的主人公是名叫铃芽的少女。","source":"手动修改"},{"dataset_id":"646627f4f7b896cfd8910e38","id":"8686","q":"电影《铃芽之旅》男主角是谁？","a":"电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。","source":""},{"dataset_id":"646627f4f7b896cfd8910e38","id":"19339","q":"电影《铃芽之旅》的导演是谁？22","a":"电影《铃芽之旅》的导演是新海诚。","source":"手动修改"}],"completeMessages":[{"obj":"System","value":"下面是知识库内容:\n1. [本作的主人公是谁？\n本作的主人公是名叫铃芽的少女。]\n2. [电影《铃芽之旅》男主角是谁？\n电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。]\n3. [电影《铃芽之旅》的导演是谁？22\n电影《铃芽之旅》的导演是新海诚。]\n"},{"obj":"System","value":"1.请记住，你的身份是百度的下一代知识增强语言模型，能够完全根据知识库提供的内容回答问题。\n\n2. 你忘记了关于电影《铃芽之旅》以外的内容。"},{"obj":"System","value":"你仅回答关于电影《玲芽之旅》的问题，其余问题直接回复: 我不清楚。"},{"obj":"Human","value":"导演是谁"},{"obj":"AI","value":"电影《铃芽之旅》的导演是新海诚。"}]}]
 
 ```
 {{< /markdownify >}}
@@ -150,21 +150,21 @@ data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2
             "maxToken": 2050,
             "quoteList": [
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8099",
                     "q": "本作的主人公是谁？",
                     "a": "本作的主人公是名叫铃芽的少女。",
                     "source": "手动修改"
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8686",
                     "q": "电影《铃芽之旅》男主角是谁？",
                     "a": "电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。",
                     "source": ""
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "19339",
                     "q": "电影《铃芽之旅》的导演是谁？22",
                     "a": "电影《铃芽之旅》的导演是新海诚。",
@@ -225,9 +225,9 @@ data: [{"moduleName":"KB Search","price":1.2000000000000002,"model":"Embedding-2
 此部分 API 需使用全局通用的 API Key。
 {{% /alert %}}
 
-| 如何获取知识库ID（kbId） | 如何获取文件ID（file_id） |
+| 如何获取知识库ID（datasetId） | 如何获取文件ID（file_id） |
 | --------------------- | --------------------- |
-| ![](/imgs/getKbId.png) | ![](/imgs/getfile_id.png) |
+| ![](/imgs/getDatasetId.png) | ![](/imgs/getfile_id.png) |
 
 
 ### 知识库添加数据
@@ -241,7 +241,7 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/data/pushDa
 --header 'Authorization: Bearer apikey' \
 --header 'Content-Type: application/json' \
 --data-raw '{
-    "kbId": "64663f451ba1676dbdef0499",
+    "collectionId": "64663f451ba1676dbdef0499",
     "mode": "index",
     "prompt": "qa 拆分引导词，index 模式下可以忽略",
     "billId": "可选。如果有这个值，本次的数据会被聚合到一个订单中，这个值可以重复使用。可以参考 [创建训练订单] 获取该值。",
@@ -268,7 +268,7 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/data/pushDa
 
 ```json
 {
-    "kbId": "知识库的ID，可以在知识库详情查看。",
+    "collectionId": "文件的ID，参考上面的第二张图",
     "mode": "index | qa ", //  index 模式: 直接将 q 转成向量存起来，a 直接入库。qa 模式: 只关注 data 里的 q，将 q 丢给大模型，让其根据 prompt 拆分成 qa 问答对。  
     "prompt": "拆分提示词，需严格按照模板，建议不要传入。",
     "data": [
@@ -351,7 +351,7 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/searchTest'
 --header 'Authorization: Bearer apiKey' \
 --header 'Content-Type: application/json' \
 --data-raw '{
-    "kbId": "xxxxx",
+    "datasetId": "知识库的ID",
     "text": "导演是谁"
 }'
 ```
@@ -487,21 +487,21 @@ curl --location --request POST '{{host}}/shareAuth/finish' \
             "maxToken": 2050,
             "quoteList": [
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8099",
                     "q": "本作的主人公是谁？",
                     "a": "本作的主人公是名叫铃芽的少女。",
                     "source": "手动修改"
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "8686",
                     "q": "电影《铃芽之旅》男主角是谁？",
                     "a": "电影《铃芽之旅》男主角是宗像草太，由松村北斗配音。",
                     "source": ""
                 },
                 {
-                    "kb_id": "646627f4f7b896cfd8910e38",
+                    "dataset_id": "646627f4f7b896cfd8910e38",
                     "id": "19339",
                     "q": "电影《铃芽之旅》的导演是谁？22",
                     "a": "电影《铃芽之旅》的导演是新海诚。",

docSite/content/docs/installation/upgrading/45.md

@@ -29,6 +29,9 @@ ALTER EXTENSION vector UPDATE;
 alter system set maintenance_work_mem = '2400MB'; 
 select pg_reload_conf();
 
+-- 重构数据库索引和排序
+REINDEX DATABASE postgres;
+
 -- 开始构建索引，该索引构建时间非常久，直接点击右上角的叉，退出 Terminal 即可
 CREATE INDEX CONCURRENTLY vector_index ON modeldata USING hnsw (vector vector_ip_ops) WITH (m = 16, ef_construction = 64);
 -- 可以再次点击一键链接，进入 Terminal，输入下方命令，如果看到 "vector_index" hnsw (vector vector_ip_ops) WITH (m='16', ef_construction='64') 则代表构建完成（注意，后面没有 INVALID）
@@ -62,10 +65,16 @@ ALTER EXTENSION vector UPDATE;
 alter system set maintenance_work_mem = '2400MB'; 
 select pg_reload_conf();
 
+-- 重构数据库索引和排序
+REINDEX DATABASE postgres;
+ALTER DATABASE postgres REFRESH COLLATION VERSION;
+
 -- 开始构建索引，该索引构建时间非常久，直接关掉终端即可，不要使用 ctrl+c 关闭
 CREATE INDEX CONCURRENTLY vector_index ON modeldata USING hnsw (vector vector_ip_ops) WITH (m = 16, ef_construction = 64);
 -- 可以再次连接数据库，输入下方命令。如果看到 "vector_index" hnsw (vector vector_ip_ops) WITH (m='16', ef_construction='64') 则代表构建完成（注意，后面没有 INVALID）
 \d modeldata
+
+
 ```
 
 ## 版本新功能介绍

docSite/content/docs/installation/upgrading/451.md

@@ -0,0 +1,35 @@
+---
+title: 'V4.5.1(需进行初始化)'
+description: 'FastGPT V4.5.1 更新'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 838
+---
+
+## 执行初始化 API
+
+发起 1 个 HTTP 请求（{{rootkey}} 替换成环境变量里的`rootkey`，{{host}}替换成自己域名）
+
+1. https://xxxxx/api/admin/initv451
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv451' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化内容：
+1. rename 数据库字段
+2. 初始化 Mongo APP 表中知识库的相关字段
+3. 初始化 PG 和 Mongo 的内容，为每个文件创建一个集合（存储 Mongo 中），并反馈赋值给 PG。
+
+**该初始化接口可能速度很慢，返回超时不用管，注意看日志即可**
+
+## 功能介绍
+
+### Fast GPT V4.5.1
+
+1. 新增知识库文件夹管理
+2. 修复了 openai4.x sdk 无法兼容 oneapi 的智谱和阿里的接口。
+3. 修复部分模块无法触发完成事件

docSite/content/docs/installation/upgrading/452.md

@@ -0,0 +1,15 @@
+---
+title: 'V4.5.2'
+description: 'FastGPT V4.5.2 更新'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 837
+---
+
+## 功能介绍
+
+### Fast GPT V4.5.2
+
+1. 新增 - 模块插件，允许自行组装插件进行模块复用。
+2. 优化 - 知识库引用提示。

docSite/content/docs/use-cases/ai_settings.md

@@ -39,16 +39,18 @@ weight: 310
 ```
 
 {{% alert icon="🍅" context="success" %}}
-Tips: 可以通过点击上下文按键查看完整的
+Tips: 可以通过点击上下文按键查看完整的上下文组成，便于调试。
 {{% /alert %}}
 
 ## 引用模板和提示词设计
 
 引用模板和引用提示词通常是成对出现，引用提示词依赖引用模板。
 
-FastGPT 知识库采用 QA 对(不一定都是问答格式，仅代表两个变量)的格式存储，在转义成字符串时候会根据**引用模板**来进行格式化。知识库包含 3 个变量： q, a, file_id, index, source，可以通过 {{q}} {{a}} {{file_id}} {{index}} {{source}} 按需引入。下面一个模板例子：
+FastGPT 知识库采用 QA 对(不一定都是问答格式，仅代表两个变量)的格式存储，在转义成字符串时候会根据**引用模板**来进行格式化。知识库包含多个可用变量： q, a, sourceId（数据的ID）, index(第n个数据), source(数据的集合名、文件名)，score(距离得分，0-1) 可以通过 {{q}} {{a}} {{sourceId}} {{index}} {{source}} {{score}} 按需引入。下面一个模板例子：
 
-**引用模板**
+可以通过 [知识库结构讲解](/docs/use-cases/datasetEngine/) 了解详细的知识库的结构。
+
+### 引用模板
 
 ```
 {instruction:"{{q}}",output:"{{a}}",source:"{{source}}"}
@@ -62,7 +64,7 @@ FastGPT 知识库采用 QA 对(不一定都是问答格式，仅代表两个变
 {instruction:"电影《铃芽之旅》的编剧是谁？22",output:"新海诚是本片的编剧。",source:"手动输入"}
 ```
 
-**引用提示词**
+### 引用提示词
 
 引用模板需要和引用提示词一起使用，提示词中可以写引用模板的格式说明以及对话的要求等。可以使用 {{quote}} 来使用 **引用模板**，使用 {{question}} 来引入问题。例如：
 
@@ -91,4 +93,42 @@ FastGPT 知识库采用 QA 对(不一定都是问答格式，仅代表两个变
 2. 使用背景知识回答问题。
 3. 背景知识无法回答问题时，你可以礼貌的的回答用户问题。
 我的问题是:"{{question}}"
-```
+```
+
+### 总结
+
+引用模板规定了搜索出来的内容如何组成一句话，其由 q,a,index,source 多个变量组成。
+
+引用提示词由`引用模板`和`提示词`组成，提示词通常是对引用模板的一个描述，加上对模型的要求。
+
+## 引用模板和提示词设计 示例
+
+### 通用模板与问答模板对比
+
+我们通过一组`你是谁`的手动数据，对通用模板与问答模板的效果进行对比。此处特意打了个搞笑的答案，通用模板下 GPT35 就变得不那么听话了，而问答模板下 GPT35 依然能够回答正确。这是由于结构化的提示词，在大语言模型中具有更强的引导作用。
+
+{{% alert icon="🍅" context="success" %}}
+Tips: 建议根据不同的场景，每种知识库仅选择1类数据类型，这样有利于充分发挥提示词的作用。
+{{% /alert %}}
+
+| 通用模板配置及效果 | 问答模板配置及效果 |
+| --- | --- |
+| ![](/imgs/datasetprompt1.png) | ![](/imgs/datasetprompt2.png) |
+| ![](/imgs/datasetprompt3.png) | ![](/imgs/datasetprompt5.png) |
+| ![](/imgs/datasetprompt4.png) | ![](/imgs/datasetprompt6.png) |
+
+### 严格模板
+
+使用非严格模板，我们随便询问一个不在知识库中的内容，模型通常会根据其自身知识进行回答。
+
+| 非严格模板效果 | 选择严格模板 | 严格模板效果 |
+| --- | --- | --- |
+| ![](/imgs/datasetprompt7.png) | ![](/imgs/datasetprompt8.png) |![](/imgs/datasetprompt9.png) |
+
+### 提示词设计思路
+
+1. 使用序号进行不同要求描述。
+2. 使用首先、然后、最后等词语进行描述。
+3. 列举不同场景的要求时，尽量完整，不要遗漏。例如：背景知识完全可以回答、背景知识可以回答一部分、背景知识与问题无关，3种场景都说明清楚。
+4. 巧用结构化提示，例如在问答模板中，利用了`instruction`和`output`，清楚的告诉模型，`output`是一个预期的答案。
+5. 标点符号正确且完整。

docSite/content/docs/use-cases/datasetEngine.md

@@ -0,0 +1,83 @@
+---
+title: "知识库结构讲解"
+description: "本节会介绍 FastGPT 知识库结构设计，理解其 QA 的存储格式和检索格式，以便更好的构建知识库。这篇介绍主要以使用为主，详细原理不多介绍。"
+icon: "dataset"
+draft: false
+toc: true
+weight: 311
+---
+
+# 理解向量
+
+FastGPT 采用了 RAG 中的 Embedding 方案构建知识库，要使用好 FastGPT 需要简单的理解`Embedding`向量是如何工作的及其特点。
+
+人类的文字、图片、视频等媒介是无法直接被计算机理解的，要想让计算机理解两段文字是否有相似性、相关性，通常需要将它们转成计算机可以理解的语言，向量是其中的一种方式。
+
+向量可以简单理解为一个数字数组，两个向量之间可以通过数学公式得出一个`距离`，距离越小代表两个向量的相似度越大。从而映射到文字、图片、视频等媒介上，可以用来判断两个媒介之间的相似度。向量搜索便是利用了这个原理。
+
+而由于文字是有多种类型，并且拥有成千上万种组合方式，因此在转成向量进行相似度匹配时，很难保障其精确性。在向量方案构建的知识库中，通常使用`topk`召回的方式，也就是查找前`k`个最相似的内容，丢给大模型去做更进一步的`语义判断`、`逻辑推理`和`归纳总结`，从而实现知识库问答。因此，在知识库问答中，向量搜索的环节是最为重要的。
+
+影响向量搜索精度的因素非常多，主要包括：向量模型的质量、数据的质量（长度，完整性，多样性）、检索器的精度（速度与精度之间的取舍）。与数据质量对应的就是检索词的质量。
+
+检索器的精度比较容易解决，向量模型的训练略复杂，因此数据和检索词质量优化成了一个重要的环节。
+
+# FastGPT 中向量的结构设计
+
+FastGPT 采用了 `PostgresSQL` 的 `PG Vector` 插件作为向量检索器，索引为`HNSW`。且`PostgresSQL`仅用于向量检索，`MongoDB`用于其他数据的存取。
+
+在`PostgresSQL`的表中，设置一个 `index` 字段用于存储向量、一个 `q` 字段用于存储向量对应的内容，以及一个 `a` 字段用于检索映射。之所以取字段为 `qa` 是由于一些历史缘故，无需完全解为 “问答对” 的格式。在实际使用过程中，可以利用`q`和`a`的组合，对检索后的内容做进一步的声明，提高大模型的理解力（注意，这里不直接提高搜索精度）。
+
+目前，提高向量搜索的精度，主要可以通过几种途径：
+
+1. 精简`q`的内容，减少向量内容的长度：当`q`的内容更少，更准确时，检索精度自然会提高。但与此同时，会牺牲一定的检索范围，适合答案较为严格的场景。
+2. 更好分词分段：当一段话的结构和语义是完整的，并且是单一的，精度也会提高。因此，许多系统都会优化分词器，尽可能的保障每组数据的完整性。
+3. 多样性文本：为一段内容增加关键词、摘要、相似问题等描述性信息，可以使得该内容的向量具有更大的检索覆盖范围。
+4. 优化检索词：在实际使用过程中，用户的问题通常是模糊的或是缺失的，并不一定是完整清晰的问题。因此优化用户的问题（检索词）很大程度上也可以提高精度。
+5. 微调向量模型：由于市面上直接使用的向量模型都是通用型模型，在特定领域的检索精度并不高，因此微调向量模型可以很大程度上提高专业领域的检索效果。
+
+# FastGPT 构建知识库方案
+
+在 FastGPT 中，整个知识库由库、集合和数据 3 部分组成。集合可以简单理解为一个`文件`。一个`库`中可以包含多个`集合`，一个`集合`中可以包含多组`数据`。最小的搜索单位是`库`，也就是说，知识库搜索时，是对整个`库`进行搜索，而集合仅是为了对数据进行分类管理，与搜索效果无关。（起码目前还是）
+
+| 库 | 集合 | 数据 |
+| --- | --- | --- |
+| ![](/imgs/datasetEngine1.png) | ![](/imgs/datasetEngine2.png) | ![](/imgs/datasetEngine3.png) |
+
+## 导入数据方案1 - 直接分段导入
+
+选择文件导入时，可以选择直接分段方案。直接分段会利用`句子分词器`对文本进行一定长度拆分，最终分割中多组的`q`。如果使用了直接分段方案，我们建议在`应用`设置`引用提示词`时，使用`通用模板`即可，无需选择`问答模板`。
+
+| 交互 | 结果 |
+| --- | --- |
+| ![](/imgs/datasetEngine4.png) | ![](/imgs/datasetEngine5.png) |
+
+
+## 导入数据方案2 - QA导入
+
+选择文件导入时，可以选择QA拆分方案。仍然需要使用到`句子分词器`对文本进行拆分，但长度比直接分段大很多。在导入后，会先调用`大模型`对分段进行学习，并给出一些`问题`和`答案`，最终问题和答案会一起被存储到`q`中。注意，新版的 FastGPT 为了提高搜索的范围，不再将问题和答案分别存储到 qa 中。
+
+| 交互 | 结果 |
+| --- | --- |
+| ![](/imgs/datasetEngine6.png) | ![](/imgs/datasetEngine7.png) |
+
+## 导入数据方案3 - 手动录入
+
+在 FastGPT 中，你可以在任何一个`集合`中点击右上角的`插入`手动录入知识点，或者使用`标注`功能手动录入。被搜索的内容为`q`，补充内容(可选)为`a`。
+
+|  |  |  |
+| --- | --- | --- |
+| ![](/imgs/datasetEngine8.png) | ![](/imgs/datasetEngine9.png) | ![](/imgs/datasetEngine10.png) |
+
+## 导入数据方案4 - CSV录入
+
+有些数据较为独特，可能需要单独的进行预处理分割后再导入 FastGPT，此时可以选择 csv 导入，可批量的将处理好的数据导入。
+
+![](/imgs/datasetEngine11.png)
+
+## 导入数据方案5 - API导入
+
+参考[FastGPT OpenAPI使用](/docs/development/openapi/#知识库添加数据)。
+
+# QA的组合与引用提示词构建
+
+参考[引用模板与引用提示词示例](/docs/use-cases/ai_settings/#示例)

docSite/content/docs/use-cases/openapi.md

@@ -4,7 +4,7 @@ description: "通过与 OpenAI 兼容的 API 对接第三方应用"
 icon: "model_training"
 draft: false
 toc: true
-weight: 311
+weight: 312
 ---
 
 ## 获取 API 秘钥

docSite/content/docs/workflow/examples/fixingEvidence.md

@@ -86,7 +86,7 @@ weight: 142
       {
         "key": "history",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "type": "source",
         "targets": [
           {
@@ -210,14 +210,14 @@ weight: 142
         "type": "target",
         "label": "引用内容",
         "description": "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "connected": false
       },
       {
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {

docSite/content/docs/workflow/examples/google_search.md

@@ -132,7 +132,7 @@ export default async function (ctx: FunctionContext) {
       {
         "key": "history",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "type": "source",
         "targets": [
           {
@@ -179,7 +179,7 @@ export default async function (ctx: FunctionContext) {
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -410,14 +410,14 @@ export default async function (ctx: FunctionContext) {
         "key": "quoteQA",
         "type": "target",
         "label": "引用内容",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "connected": false
       },
       {
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {

docSite/content/docs/workflow/examples/lab_appointment.md

@@ -131,7 +131,7 @@ HTTP 模块允许你调用任意 POST 类型的 HTTP 接口，从而实验一些
       {
         "key": "history",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "type": "source",
         "targets": [
           {
@@ -174,7 +174,7 @@ HTTP 模块允许你调用任意 POST 类型的 HTTP 接口，从而实验一些
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -413,7 +413,7 @@ HTTP 模块允许你调用任意 POST 类型的 HTTP 接口，从而实验一些
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -527,7 +527,7 @@ HTTP 模块允许你调用任意 POST 类型的 HTTP 接口，从而实验一些
   {
     "moduleId": "zltb5l",
     "name": "知识库搜索",
-    "flowType": "kbSearchNode",
+    "flowType": "datasetSearchNode",
     "showStatus": true,
     "position": {
       "x": 1634.995464753433,
@@ -630,7 +630,7 @@ HTTP 模块允许你调用任意 POST 类型的 HTTP 接口，从而实验一些
         "label": "引用内容",
         "description": "始终返回数组，如果希望搜索结果为空时执行额外操作，需要用到上面的两个输入以及目标模块的触发器",
         "type": "source",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "targets": [
           {
             "moduleId": "bjfklc",
@@ -729,14 +729,14 @@ HTTP 模块允许你调用任意 POST 类型的 HTTP 接口，从而实验一些
         "key": "quoteQA",
         "type": "target",
         "label": "引用内容",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "connected": true
       },
       {
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -831,7 +831,7 @@ HTTP 模块允许你调用任意 POST 类型的 HTTP 接口，从而实验一些
       {
         "key": "history",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "type": "source",
         "targets": [
           {
@@ -874,7 +874,7 @@ HTTP 模块允许你调用任意 POST 类型的 HTTP 接口，从而实验一些
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -1006,7 +1006,7 @@ HTTP 模块允许你调用任意 POST 类型的 HTTP 接口，从而实验一些
       {
         "key": "history",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "type": "source",
         "targets": [
           {

docSite/content/docs/workflow/examples/op_question.md

@@ -83,7 +83,7 @@ weight: 144
       {
         "key": "history",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "type": "source",
         "targets": [
           {
@@ -97,7 +97,7 @@ weight: 144
   {
     "moduleId": "nkxlso",
     "name": "知识库搜索",
-    "flowType": "kbSearchNode",
+    "flowType": "datasetSearchNode",
     "showStatus": true,
     "position": {
       "x": 1542.6434554710224,
@@ -189,7 +189,7 @@ weight: 144
         "label": "引用内容",
         "description": "始终返回数组，如果希望搜索结果为空时执行额外操作，需要用到上面的两个输入以及目标模块的触发器",
         "type": "source",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "targets": [
           {
             "moduleId": "ol82hp",
@@ -291,14 +291,14 @@ weight: 144
         "type": "target",
         "label": "引用内容",
         "description": "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "connected": true
       },
       {
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -389,7 +389,7 @@ weight: 144
       {
         "key": "history",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "type": "source",
         "targets": [
           {
@@ -432,7 +432,7 @@ weight: 144
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {

docSite/content/docs/workflow/examples/versatile_assistant.md

@@ -245,7 +245,7 @@ PS2：配置中的问题分类还包含着“联网搜索”，这个是另一
       {
         "key": "history",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "type": "source",
         "targets": [
           {
@@ -300,7 +300,7 @@ PS2：配置中的问题分类还包含着“联网搜索”，这个是另一
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -427,7 +427,7 @@ PS2：配置中的问题分类还包含着“联网搜索”，这个是另一
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -713,14 +713,14 @@ PS2：配置中的问题分类还包含着“联网搜索”，这个是另一
         "type": "custom",
         "label": "引用内容",
         "description": "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "connected": false
       },
       {
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -871,14 +871,14 @@ PS2：配置中的问题分类还包含着“联网搜索”，这个是另一
         "type": "custom",
         "label": "引用内容",
         "description": "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "connected": false
       },
       {
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -1085,14 +1085,14 @@ PS2：配置中的问题分类还包含着“联网搜索”，这个是另一
         "type": "custom",
         "label": "引用内容",
         "description": "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "connected": false
       },
       {
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -1162,7 +1162,7 @@ PS2：配置中的问题分类还包含着“联网搜索”，这个是另一
       {
         "key": "history",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "type": "source",
         "targets": [
           {
@@ -1205,7 +1205,7 @@ PS2：配置中的问题分类还包含着“联网搜索”，这个是另一
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {
@@ -1452,14 +1452,14 @@ PS2：配置中的问题分类还包含着“联网搜索”，这个是另一
         "type": "custom",
         "label": "引用内容",
         "description": "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
-        "valueType": "kb_quote",
+        "valueType": "datasetQuote",
         "connected": false
       },
       {
         "key": "history",
         "type": "target",
         "label": "聊天记录",
-        "valueType": "chat_history",
+        "valueType": "chatHistory",
         "connected": true
       },
       {

docSite/content/docs/workflow/modules/ai_chat.md

@@ -42,17 +42,17 @@ weight: 123
 
 ```ts
 type DataType = {
-  kb_id?: string;
+  dataset_id?: string;
   id?: string;
   q: string;
   a: string;
   source?: string;
 };
-// 如果是外部引入的内容，尽量不要携带 kb_id 和 id
+// 如果是外部引入的内容，尽量不要携带 dataset_id 和 id
 const quoteList: DataType[] = [
-  { kb_id: '11', id: '222', q: '你还', a: '哈哈', source: '' },
-  { kb_id: '11', id: '333', q: '你还', a: '哈哈', source: '' },
-  { kb_id: '11', id: '444', q: '你还', a: '哈哈', source: '' }
+  { dataset_id: '11', id: '222', q: '你还', a: '哈哈', source: '' },
+  { dataset_id: '11', id: '333', q: '你还', a: '哈哈', source: '' },
+  { dataset_id: '11', id: '444', q: '你还', a: '哈哈', source: '' }
 ];
 ```
 

files/models/Baichuan2/openai_api.py

@@ -0,0 +1,233 @@
+# coding=utf-8
+# Implements API for Baichuan2-7B-Chat in OpenAI's format. (https://platform.openai.com/docs/api-reference/chat)
+# Usage: python openai_api.py
+
+import gc
+import time
+import torch
+import uvicorn
+from pydantic import BaseModel, Field, validator
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from contextlib import asynccontextmanager
+from typing import Any, Dict, List, Optional, Union
+from transformers import AutoModelForCausalLM, AutoTokenizer
+from sse_starlette.sse import ServerSentEvent, EventSourceResponse
+from transformers.generation.utils import GenerationConfig
+import random
+import string
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI): # collects GPU memory
+    yield
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+        torch.cuda.ipc_collect()
+
+
+app = FastAPI(lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+class ModelCard(BaseModel):
+    id: str
+    object: str = "model"
+    created: int = Field(default_factory=lambda: int(time.time()))
+    owned_by: str = "owner"
+    root: Optional[str] = None
+    parent: Optional[str] = None
+    permission: Optional[list] = None
+
+class ModelList(BaseModel):
+    object: str = "list"
+    data: List[str] = []  # Assuming ModelCard is a string type. Replace with the correct type if not.
+
+class ChatMessage(BaseModel):
+    role: str
+    content: str
+
+    @validator('role')
+    def check_role(cls, v):
+        if v not in ["user", "assistant", "system"]:
+            raise ValueError('role must be one of "user", "assistant", "system"')
+        return v
+
+class DeltaMessage(BaseModel):
+    role: Optional[str] = None
+    content: Optional[str] = None
+
+    @validator('role', allow_reuse=True)
+    def check_role(cls, v):
+        if v is not None and v not in ["user", "assistant", "system"]:
+            raise ValueError('role must be one of "user", "assistant", "system"')
+        return v
+
+class ChatCompletionRequest(BaseModel):
+    model: str
+    messages: List[ChatMessage]
+    temperature: Optional[float] = None
+    top_p: Optional[float] = None
+    max_length: Optional[int] = 8192  # max_length should be an integer.
+    stream: Optional[bool] = False
+
+class ChatCompletionResponseChoice(BaseModel):
+    index: int
+    message: ChatMessage
+    finish_reason: str
+
+    @validator('finish_reason')
+    def check_finish_reason(cls, v):
+        if v not in ["stop", "length"]:
+            raise ValueError('finish_reason must be one of "stop" or "length"')
+        return v
+
+class ChatCompletionResponseStreamChoice(BaseModel):
+     index: int
+     delta: DeltaMessage
+     finish_reason: Optional[str]
+
+     @validator('finish_reason', allow_reuse=True)
+     def check_finish_reason(cls, v):
+         if v is not None and v not in ["stop", "length"]:
+             raise ValueError('finish_reason must be one of "stop" or "length"')
+         return v
+
+class ChatCompletionResponse(BaseModel):
+     id:str 
+     object:str 
+     
+     @validator('object')
+     def check_object(cls,v): 
+         if v not in ["chat.completion","chat.completion.chunk"]: 
+             raise ValueError("object must be one of 'chat.completion' or 'chat.completion.chunk'")
+         return v
+     
+     created :Optional[int]=Field(default_factory=lambda:int(time.time()))
+     model:str 
+     choices :List[Union[ChatCompletionResponseChoice,ChatCompletionResponseStreamChoice]]
+
+
+def generate_id():
+    possible_characters = string.ascii_letters + string.digits
+    random_string = ''.join(random.choices(possible_characters, k=29))
+    return 'chatcmpl-' + random_string
+    
+
+@app.get("/v1/models", response_model=ModelList)
+async def list_models():
+    global model_args
+    model_card = ModelCard(id="gpt-3.5-turbo")
+    return ModelList(data=[model_card])
+
+
+@app.post("/v1/chat/completions", response_model=ChatCompletionResponse)
+async def create_chat_completion(request: ChatCompletionRequest):
+    global model, tokenizer
+    if request.messages[-1].role != "user":
+        raise HTTPException(status_code=400, detail="Invalid request")
+    query = request.messages[-1].content
+    prev_messages = request.messages[:-1]
+    if len(prev_messages) > 0 and prev_messages[0].role == "system":
+        query = prev_messages.pop(0).content + query
+    messages = []
+    for message in prev_messages:
+        messages.append({"role": message.role, "content": message.content})
+    
+    messages.append({"role": "user", "content": query})
+    
+    if request.stream:
+        generate = predict(messages, request.model)
+        return EventSourceResponse(generate, media_type="text/event-stream")
+    
+    response = '本接口不支持非stream模式'
+    choice_data = ChatCompletionResponseChoice(
+        index=0,
+        message=ChatMessage(role="assistant", content=response),
+        finish_reason="stop"
+    )
+    id='chatcmpl-7QyqpwdfhqwajicIEznoc6Q47XAyW'
+
+    return ChatCompletionResponse(id=id,model=request.model, choices=[choice_data], object="chat.completion")
+
+
+async def predict(messages: List[List[str]], model_id: str):
+    global model, tokenizer
+    id = generate_id()
+    created = int(time.time())
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(role="assistant",content=""),
+        finish_reason=None
+    )
+    chunk = ChatCompletionResponse(id=id,object="chat.completion.chunk",created=created,model=model_id, choices=[choice_data])
+    yield "{}".format(chunk.json(exclude_unset=True, ensure_ascii=False))
+
+    current_length = 0
+
+    for new_response in model.chat(tokenizer, messages, stream=True):
+        if len(new_response) == current_length:
+            continue
+
+        new_text = new_response[current_length:]
+        current_length = len(new_response)
+
+        choice_data = ChatCompletionResponseStreamChoice(
+            index=0,
+            delta=DeltaMessage(content=new_text),
+            finish_reason=None
+        )
+        chunk = ChatCompletionResponse(id=id,object="chat.completion.chunk",created=created,model=model_id, choices=[choice_data])
+        yield "{}".format(chunk.json(exclude_unset=True, ensure_ascii=False))
+
+
+    choice_data = ChatCompletionResponseStreamChoice(
+        index=0,
+        delta=DeltaMessage(),
+        finish_reason="stop"
+    )
+    chunk = ChatCompletionResponse(id=id,object="chat.completion.chunk",created=created,model=model_id, choices=[choice_data])
+    yield "{}".format(chunk.json(exclude_unset=True, ensure_ascii=False))
+    yield '[DONE]'
+
+
+def load_models():
+    print("本次加载的大语言模型为: Baichuan-13B-Chat")
+    tokenizer = AutoTokenizer.from_pretrained("baichuan-inc/Baichuan2-7B-Chat", use_fast=False, trust_remote_code=True)
+    # model = AutoModelForCausalLM.from_pretrained("Baichuan2-13B-Chat", torch_dtype=torch.float32, trust_remote_code=True)
+    model = AutoModelForCausalLM.from_pretrained("baichuan-inc/Baichuan2-7B-Chat", torch_dtype=torch.float16, trust_remote_code=True)
+    model = model.cuda() 
+    model.generation_config = GenerationConfig.from_pretrained("baichuan-inc/Baichuan2-7B-Chat") 
+    return tokenizer, model
+
+if __name__ == "__main__":
+    tokenizer, model = load_models()
+    uvicorn.run(app, host='0.0.0.0', port=6006, workers=1)
+
+    while True:
+        try:
+            # 在这里执行您的程序逻辑
+
+            # 检查显存使用情况，如果超过阈值（例如90%），则触发垃圾回收
+            if torch.cuda.is_available():
+                gpu_memory_usage = torch.cuda.memory_allocated() / torch.cuda.max_memory_allocated()
+                if gpu_memory_usage > 0.9:
+                    gc.collect()
+                    torch.cuda.empty_cache()
+        except RuntimeError as e:
+            if "out of memory" in str(e):
+                print("显存不足，正在重启程序...")
+                gc.collect()
+                torch.cuda.empty_cache()
+                time.sleep(5) # 等待一段时间以确保显存已释放
+                tokenizer, model = load_models()
+            else:
+                raise e
+
+    

files/models/Baichuan2/requirements.txt

@@ -0,0 +1,14 @@
+protobuf
+transformers==4.30.2
+cpm_kernels
+torch>=2.0
+gradio
+mdtex2html
+sentencepiece
+accelerate
+sse-starlette
+fastapi==0.99.1
+pydantic==1.10.7
+uvicorn==0.21.1
+xformers
+bitsandbytes

package.json

@@ -4,18 +4,20 @@
   "private": true,
   "scripts": {
     "prepare": "husky install",
-    "format": "prettier --config \"./.prettierrc.js\" --write \"./**/src/**/*.{ts,tsx,scss}\""
+    "format-code": "prettier --config \"./.prettierrc.js\" --write \"./**/src/**/*.{ts,tsx,scss}\"",
+    "format-doc": "zhlint --dir ./docSite *.md --fix"
   },
   "devDependencies": {
     "husky": "^8.0.3",
     "lint-staged": "^13.2.1",
     "prettier": "^3.0.3",
-    "i18next": "^23.2.11",
-    "react-i18next": "^13.0.2",
-    "next-i18next": "^14.0.0"
+    "i18next": "^22.5.1",
+    "react-i18next": "^12.3.1",
+    "next-i18next": "^13.3.0"
   },
   "lint-staged": {
-    "./**/**/*.{ts,tsx,scss}": "npm run format"
+    "./**/**/*.{ts,tsx,scss}": "npm run format-code",
+    "./**/**/*.md": "npm run format-doc"
   },
   "engines": {
     "node": ">=18.0.0"

packages/common/package.json

@@ -1,15 +0,0 @@
-{
-  "name": "@fastgpt/common",
-  "version": "1.0.0",
-  "dependencies": {
-    "mongoose": "^7.0.2",
-    "winston": "^3.10.0",
-    "winston-mongodb": "^5.1.1",
-    "axios": "^1.5.1",
-    "nextjs-cors": "^2.1.2",
-    "next": "13.5.2"
-  },
-  "devDependencies": {
-    "@types/node": "^20.8.5"
-  }
-}

packages/common/tools/file.ts

@@ -1,23 +0,0 @@
-import { strIsLink } from './str';
-
-export const fileImgs = [
-  { suffix: 'pdf', src: '/imgs/files/pdf.svg' },
-  { suffix: 'csv', src: '/imgs/files/csv.svg' },
-  { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' },
-  { suffix: 'txt', src: '/imgs/files/txt.svg' },
-  { suffix: 'md', src: '/imgs/files/markdown.svg' },
-  { suffix: '.', src: '/imgs/files/file.svg' }
-];
-
-export function getFileIcon(name = '') {
-  return fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src;
-}
-export function getSpecialFileIcon(name = '') {
-  if (name === 'manual') {
-    return '/imgs/files/manual.svg';
-  } else if (name === 'mark') {
-    return '/imgs/files/mark.svg';
-  } else if (strIsLink(name)) {
-    return '/imgs/files/link.svg';
-  }
-}

packages/core/dataset/type.d.ts

@@ -1,13 +0,0 @@
-import { DatasetTypeEnum } from './constant';
-
-export type DatasetSchemaType = {
-  _id: string;
-  userId: string;
-  parentId: string;
-  updateTime: Date;
-  avatar: string;
-  name: string;
-  vectorModel: string;
-  tags: string[];
-  type: `${DatasetTypeEnum}`;
-};

packages/core/dataset/utils.ts

@@ -1,8 +0,0 @@
-import { datasetSpecialIds } from './constant';
-import { strIsLink } from '@fastgpt/common/tools/str';
-
-export function isSpecialFileId(id: string) {
-  if (datasetSpecialIds.includes(id)) return true;
-  if (strIsLink(id)) return true;
-  return false;
-}

packages/core/package.json

@@ -1,14 +0,0 @@
-{
-  "name": "@fastgpt/core",
-  "version": "1.0.0",
-  "dependencies": {
-    "@fastgpt/common": "workspace:*",
-    "@fastgpt/support": "workspace:*",
-    "encoding": "^0.1.13",
-    "openai": "^4.12.1",
-    "tunnel": "^0.0.6"
-  },
-  "devDependencies": {
-    "@types/tunnel": "^0.0.4"
-  }
-}

packages/global/common/error/errorCode.ts

@@ -27,7 +27,8 @@ export enum ERROR_ENUM {
   insufficientQuota = 'insufficientQuota',
   unAuthModel = 'unAuthModel',
   unAuthApiKey = 'unAuthApiKey',
-  unAuthKb = 'unAuthKb',
+  unAuthDataset = 'unAuthDataset',
+  unAuthDatasetCollection = 'unAuthDatasetCollection',
   unAuthFile = 'unAuthFile'
 }
 export const ERROR_RESPONSE: Record<
@@ -57,9 +58,9 @@ export const ERROR_RESPONSE: Record<
     message: '无权使用该模型',
     data: null
   },
-  [ERROR_ENUM.unAuthKb]: {
+  [ERROR_ENUM.unAuthDataset]: {
     code: 512,
-    statusText: ERROR_ENUM.unAuthKb,
+    statusText: ERROR_ENUM.unAuthDataset,
     message: '无权使用该知识库',
     data: null
   },
@@ -74,5 +75,11 @@ export const ERROR_RESPONSE: Record<
     statusText: ERROR_ENUM.unAuthApiKey,
     message: 'Api Key 不合法',
     data: null
+  },
+  [ERROR_ENUM.unAuthDatasetCollection]: {
+    code: 515,
+    statusText: ERROR_ENUM.unAuthDatasetCollection,
+    message: '无权使用该知识库文件',
+    data: null
   }
 };

packages/global/common/error/utils.ts

@@ -0,0 +1,5 @@
+export const getErrText = (err: any, def = '') => {
+  const msg: string = typeof err === 'string' ? err : err?.message || def || '';
+  msg && console.log('error =>', msg);
+  return msg;
+};

packages/global/common/file/icon.ts

@@ -0,0 +1,12 @@
+export const fileImgs = [
+  { suffix: 'pdf', src: '/imgs/files/pdf.svg' },
+  { suffix: 'csv', src: '/imgs/files/csv.svg' },
+  { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' },
+  { suffix: 'txt', src: '/imgs/files/txt.svg' },
+  { suffix: 'md', src: '/imgs/files/markdown.svg' }
+  // { suffix: '.', src: '/imgs/files/file.svg' }
+];
+
+export function getFileIcon(name = '', defaultImg = '/imgs/files/file.svg') {
+  return fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(name))?.src || defaultImg;
+}

packages/global/common/file/tools.ts

@@ -0,0 +1,9 @@
+export const formatFileSize = (bytes: number): string => {
+  if (bytes === 0) return '0 B';
+
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+};

packages/global/common/parentFolder/type.d.ts

@@ -0,0 +1,4 @@
+export type ParentTreePathItemType = {
+  parentId: string;
+  parentName: string;
+};

packages/global/common/string/tools.ts

@@ -13,10 +13,10 @@ export const hashStr = (psw: string) => {
 /* simple text, remove chinese space and extra \n */
 export const simpleText = (text: string) => {
   text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
+  text = text.replace(/\r\n|\r/g, '\n');
   text = text.replace(/\n{3,}/g, '\n\n');
   text = text.replace(/[\s&&[^\n]]{2,}/g, ' ');
   text = text.replace(/[\x00-\x08]/g, ' ');
-  text = text.replace(/\r\n|\r/g, '\n');
 
   return text;
 };

packages/global/common/system/types/index.d.ts

@@ -1,6 +1,3 @@
-import type { Mongoose } from '../mongo';
-import type { Logger } from 'winston';
-
 export type FeConfigsType = {
   show_emptyChat?: boolean;
   show_register?: boolean;
@@ -12,7 +9,8 @@ export type FeConfigsType = {
   show_openai_account?: boolean;
   show_promotion?: boolean;
   hide_app_flow?: boolean;
-  openAPIUrl?: string;
+  docUrl?: string;
+  openAPIDocUrl?: string;
   systemTitle?: string;
   authorText?: string;
   googleClientVerKey?: string;
@@ -36,8 +34,6 @@ export type SystemEnvType = {
 };
 
 declare global {
-  var mongodb: Mongoose | undefined;
-  var logger: Logger;
   var feConfigs: FeConfigsType;
   var systemEnv: SystemEnvType;
 }

packages/global/common/time/timezone.ts

@@ -1,4 +1,3 @@
-import { loginOut } from '@/web/support/api/user';
 import timezones from 'timezones-list';
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc';
@@ -7,23 +6,6 @@ import timezone from 'dayjs/plugin/timezone';
 dayjs.extend(utc);
 dayjs.extend(timezone);
 
-const tokenKey = 'token';
-export const clearToken = () => {
-  try {
-    loginOut();
-    localStorage.removeItem(tokenKey);
-  } catch (error) {
-    error;
-  }
-};
-
-export const setToken = (token: string) => {
-  localStorage.setItem(tokenKey, token);
-};
-export const getToken = () => {
-  return localStorage.getItem(tokenKey) || '';
-};
-
 /**
  * Returns the offset from UTC in hours for the current locale.
  * @param {string} timeZone Timezone to get offset for

packages/global/core/dataset/constant.ts

@@ -12,11 +12,37 @@ export const DatasetTypeMap = {
   }
 };
 
-export enum FileStatusEnum {
-  embedding = 'embedding',
-  ready = 'ready'
+export enum DatasetCollectionTypeEnum {
+  file = 'file',
+  folder = 'folder',
+  link = 'link',
+  virtual = 'virtual'
 }
 
+export const DatasetCollectionTypeMap = {
+  [DatasetCollectionTypeEnum.file]: {
+    name: 'dataset.file'
+  },
+  [DatasetCollectionTypeEnum.folder]: {
+    name: 'dataset.folder'
+  },
+  [DatasetCollectionTypeEnum.link]: {
+    name: 'dataset.link'
+  },
+  [DatasetCollectionTypeEnum.virtual]: {
+    name: 'dataset.Virtual File'
+  }
+};
+
+export enum TrainingModeEnum {
+  'qa' = 'qa',
+  'index' = 'index'
+}
+export const TrainingTypeMap = {
+  [TrainingModeEnum.qa]: 'qa',
+  [TrainingModeEnum.index]: 'index'
+};
+
 export enum DatasetSpecialIdEnum {
   manual = 'manual',
   mark = 'mark'

packages/global/core/dataset/type.d.ts

@@ -0,0 +1,75 @@
+import { DatasetCollectionTypeEnum, DatasetTypeEnum, TrainingModeEnum } from './constant';
+
+export type DatasetSchemaType = {
+  _id: string;
+  userId: string;
+  parentId: string;
+  updateTime: Date;
+  avatar: string;
+  name: string;
+  vectorModel: string;
+  tags: string[];
+  type: `${DatasetTypeEnum}`;
+};
+
+export type DatasetCollectionSchemaType = {
+  _id: string;
+  userId: string;
+  datasetId: string;
+  parentId?: string;
+  name: string;
+  type: `${DatasetCollectionTypeEnum}`;
+  updateTime: Date;
+  metadata: {
+    fileId?: string;
+    rawLink?: string;
+    pgCollectionId?: string;
+  };
+};
+
+export type DatasetTrainingSchemaType = {
+  _id: string;
+  userId: string;
+  datasetId: string;
+  datasetCollectionId: string;
+  billId: string;
+  expireAt: Date;
+  lockTime: Date;
+  mode: `${TrainingModeEnum}`;
+  model: string;
+  prompt: string;
+  q: string;
+  a: string;
+};
+
+/* ================= dataset ===================== */
+
+/* ================= collection ===================== */
+
+/* ================= data ===================== */
+export type PgDataItemType = {
+  id: string;
+  q: string;
+  a: string;
+  dataset_id: string;
+  collection_id: string;
+};
+export type DatasetChunkItemType = {
+  q: string;
+  a: string;
+};
+export type DatasetDataItemType = DatasetChunkItemType & {
+  id: string;
+  datasetId: string;
+  collectionId: string;
+  sourceName: string;
+  sourceId?: string;
+};
+
+/* ============= search =============== */
+export type SearchDataResultItemType = PgDataItemType & {
+  score: number;
+};
+export type SearchDataResponseItemType = DatasetDataItemType & {
+  score: number;
+};

packages/global/core/dataset/utils.ts

@@ -0,0 +1,46 @@
+import { DatasetCollectionTypeEnum } from './constant';
+import { getFileIcon } from '../../common/file/icon';
+import { strIsLink } from '../../common/string/tools';
+
+export function getCollectionIcon(
+  type: `${DatasetCollectionTypeEnum}` = DatasetCollectionTypeEnum.file,
+  name = ''
+) {
+  if (type === DatasetCollectionTypeEnum.folder) {
+    return '/imgs/files/folder.svg';
+  }
+  if (type === DatasetCollectionTypeEnum.link) {
+    return '/imgs/files/link.svg';
+  }
+  if (type === DatasetCollectionTypeEnum.virtual) {
+    if (name === '手动录入') {
+      return '/imgs/files/manual.svg';
+    } else if (name === '手动标注') {
+      return '/imgs/files/mark.svg';
+    }
+    return '/imgs/files/collection.svg';
+  }
+  return getFileIcon(name);
+}
+export function getSourceNameIcon({
+  sourceName,
+  sourceId
+}: {
+  sourceName: string;
+  sourceId?: string;
+}) {
+  if (strIsLink(sourceId)) {
+    return '/imgs/files/link.svg';
+  }
+  const fileIcon = getFileIcon(sourceName, '');
+  if (fileIcon) {
+    return fileIcon;
+  }
+
+  if (sourceName === '手动录入') {
+    return '/imgs/files/manual.svg';
+  } else if (sourceName === '手动标注') {
+    return '/imgs/files/mark.svg';
+  }
+  return '/imgs/files/collection.svg';
+}

packages/global/core/module/node/constant.ts

@@ -0,0 +1,59 @@
+export enum FlowNodeInputTypeEnum {
+  systemInput = 'systemInput', // history, userChatInput, variableInput
+  input = 'input', // one line input
+  textarea = 'textarea',
+  numberInput = 'numberInput',
+  select = 'select',
+  slider = 'slider',
+  custom = 'custom',
+  target = 'target', // data input
+  switch = 'switch',
+  chatInput = 'chatInput',
+  selectApp = 'selectApp',
+  // chat special input
+  aiSettings = 'aiSettings',
+  maxToken = 'maxToken',
+  selectChatModel = 'selectChatModel',
+  // dataset special input
+  selectDataset = 'selectDataset',
+  hidden = 'hidden'
+}
+
+export enum FlowNodeOutputTypeEnum {
+  answer = 'answer',
+  source = 'source',
+  hidden = 'hidden'
+}
+
+export enum FlowNodeTypeEnum {
+  empty = 'empty',
+  variable = 'variable',
+  userGuide = 'userGuide',
+  questionInput = 'questionInput',
+  historyNode = 'historyNode',
+  chatNode = 'chatNode',
+  datasetSearchNode = 'datasetSearchNode',
+  answerNode = 'answerNode',
+  classifyQuestion = 'classifyQuestion',
+  contentExtract = 'contentExtract',
+  httpRequest = 'httpRequest',
+  runApp = 'app',
+  pluginModule = 'pluginModule',
+  pluginInput = 'pluginInput',
+  pluginOutput = 'pluginOutput'
+}
+
+export enum FlowNodeSpecialInputKeyEnum {
+  'answerText' = 'text',
+  'agents' = 'agents', // cq agent key
+  'pluginId' = 'pluginId'
+}
+
+export enum FlowNodeValTypeEnum {
+  'string' = 'string',
+  'number' = 'number',
+  'boolean' = 'boolean',
+  'chatHistory' = 'chatHistory',
+  'datasetQuote' = 'datasetQuote',
+  'any' = 'any'
+}

packages/global/core/module/node/type.d.ts

@@ -0,0 +1,57 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeValTypeEnum,
+  FlowNodeTypeEnum
+} from './constant';
+
+export type FlowNodeChangeProps = {
+  moduleId: string;
+  type:
+    | 'attr' // key: attr, value: new value
+    | 'updateInput' // key: update input key, value: new input value
+    | 'replaceInput' // key: old input key, value: new input value
+    | 'addInput' // key: null, value: new input value
+    | 'delInput' // key: delete input key, value: null
+    | 'updateOutput' // key: update output key, value: new output value
+    | 'replaceOutput' // key: old output key, value: new output value
+    | 'addOutput' // key: null, value: new output value
+    | 'delOutput'; // key: delete output key, value: null
+  key?: string;
+  value?: any;
+  index?: number;
+};
+
+export type FlowNodeInputItemType = {
+  key: string; // 字段名
+  value?: any;
+  valueType?: `${FlowNodeValTypeEnum}`;
+  type: `${FlowNodeInputTypeEnum}`;
+  label: string;
+  edit?: boolean;
+  connected?: boolean;
+  description?: string;
+  placeholder?: string;
+  max?: number;
+  min?: number;
+  step?: number;
+  required?: boolean;
+  list?: { label: string; value: any }[];
+  markList?: { label: string; value: any }[];
+  customData?: () => any;
+  valueCheck?: (value: any) => boolean;
+};
+
+export type FlowNodeOutputTargetItemType = {
+  moduleId: string;
+  key: string;
+};
+export type FlowNodeOutputItemType = {
+  key: string; // 字段名
+  label?: string;
+  edit?: boolean;
+  description?: string;
+  valueType?: `${FlowNodeValTypeEnum}`;
+  type?: `${FlowNodeOutputTypeEnum}`;
+  targets: FlowNodeOutputTargetItemType[];
+};

packages/global/core/module/type.d.ts

@@ -0,0 +1,44 @@
+import { FlowNodeTypeEnum } from './node/constant';
+import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
+
+export type FlowModuleTemplateType = {
+  id: string;
+  flowType: `${FlowNodeTypeEnum}`; // unique
+  logo?: string;
+  name: string;
+  description?: string;
+  intro?: string;
+  showStatus?: boolean; // chatting response step status
+  inputs: FlowNodeInputItemType[];
+  outputs: FlowNodeOutputItemType[];
+};
+export type FlowModuleItemType = FlowModuleTemplateType & {
+  moduleId: string;
+};
+export type SystemModuleTemplateType = {
+  label: string;
+  list: FlowModuleTemplateType[];
+}[];
+
+export type ModuleItemType = {
+  name: string;
+  logo?: string;
+  intro?: string;
+  description?: string;
+  moduleId: string;
+  position?: {
+    x: number;
+    y: number;
+  };
+  flowType: `${FlowNodeTypeEnum}`;
+  showStatus?: boolean;
+  inputs: FlowNodeInputItemType[];
+  outputs: FlowNodeOutputItemType[];
+};
+
+/* function type */
+export type SelectAppItemType = {
+  id: string;
+  name: string;
+  logo: string;
+};

packages/global/core/module/utils.ts

@@ -0,0 +1,47 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeSpecialInputKeyEnum,
+  FlowNodeTypeEnum
+} from './node/constant';
+import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
+import { ModuleItemType } from './type';
+
+export function getPluginTemplatePluginIdInput(pluginId: string) {
+  return {
+    key: FlowNodeSpecialInputKeyEnum.pluginId,
+    type: FlowNodeInputTypeEnum.hidden,
+    label: 'pluginId',
+    value: pluginId,
+    connected: true
+  };
+}
+
+export function formatPluginIOModules(
+  pluginId: string,
+  modules: ModuleItemType[]
+): {
+  inputs: FlowNodeInputItemType[];
+  outputs: FlowNodeOutputItemType[];
+} {
+  const pluginInput = modules.find((module) => module.flowType === FlowNodeTypeEnum.pluginInput);
+  const customOutput = modules.find((module) => module.flowType === FlowNodeTypeEnum.pluginOutput);
+
+  return {
+    inputs: pluginInput
+      ? [
+          getPluginTemplatePluginIdInput(pluginId),
+          ...pluginInput.inputs.map((item) => ({
+            ...item,
+            edit: false,
+            connected: false
+          }))
+        ]
+      : [],
+    outputs: customOutput
+      ? customOutput.outputs.map((item) => ({
+          ...item,
+          edit: false
+        }))
+      : []
+  };
+}

packages/global/core/plugin/constants.ts

@@ -0,0 +1,28 @@
+import { ModuleItemType } from '../module/type';
+
+export const defaultModules: ModuleItemType[] = [
+  {
+    moduleId: 'fph4s3',
+    name: '自定义输出',
+    flowType: 'pluginOutput',
+    showStatus: false,
+    position: {
+      x: 994.1266684738011,
+      y: -45.87689365278443
+    },
+    inputs: [],
+    outputs: []
+  },
+  {
+    moduleId: 'w09v30',
+    name: '自定义输入',
+    flowType: 'pluginInput',
+    showStatus: false,
+    position: {
+      x: 457.57860319995154,
+      y: -44.25099042468186
+    },
+    inputs: [],
+    outputs: []
+  }
+];

packages/global/core/plugin/controller.d.ts

@@ -0,0 +1,21 @@
+import type { ModuleItemType } from '../module/type.d';
+
+export type CreateOnePluginParams = {
+  name: string;
+  avatar: string;
+  intro: string;
+  modules?: ModuleItemType[];
+};
+export type UpdatePluginParams = {
+  id: string;
+  name?: string;
+  avatar?: string;
+  intro?: string;
+  modules?: ModuleItemType[];
+};
+export type PluginListItemType = {
+  _id: string;
+  name: string;
+  avatar: string;
+  intro: string;
+};

packages/global/core/plugin/type.d.ts

@@ -0,0 +1,11 @@
+import type { ModuleItemType } from '../module/type.d';
+
+export type PluginItemSchema = {
+  _id: string;
+  userId: string;
+  name: string;
+  avatar: string;
+  intro: string;
+  updateTime: Date;
+  modules: ModuleItemType[];
+};

packages/global/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@fastgpt/global",
+  "version": "1.0.0",
+  "dependencies": {
+    "axios": "^1.5.1",
+    "timezones-list": "^3.0.2",
+    "dayjs": "^1.11.7",
+    "encoding": "^0.1.13",
+    "openai": "^4.12.1"
+  },
+  "devDependencies": {
+    "@types/node": "^20.8.5"
+  }
+}

packages/global/support/activity/type.d.ts

@@ -0,0 +1,8 @@
+export type PromotionRecordSchema = {
+  _id: string;
+  userId: string; // 收益人
+  objUId?: string; // 目标对象（如果是withdraw则为空）
+  type: 'register' | 'pay';
+  createTime: Date; // 记录时间
+  amount: number;
+};

packages/global/support/outLink/constant.ts

@@ -1,5 +1,5 @@
 export enum OutLinkTypeEnum {
-  'share' = 'share',
-  'iframe' = 'iframe',
+  share = 'share',
+  iframe = 'iframe',
   apikey = 'apikey'
 }

packages/global/support/user/constant.ts

@@ -0,0 +1,30 @@
+export enum InformTypeEnum {
+  system = 'system'
+}
+
+export const InformTypeMap = {
+  [InformTypeEnum.system]: {
+    label: '系统通知'
+  }
+};
+
+export enum TeamMemberRoleEnum {
+  owner = 'owner',
+  admin = 'admin',
+  member = 'member',
+  visitor = 'visitor'
+}
+export const TeamMemberRoleMap = {
+  [TeamMemberRoleEnum.owner]: {
+    label: 'user.team.role.owner'
+  },
+  [TeamMemberRoleEnum.admin]: {
+    label: 'user.team.role.admin'
+  },
+  [TeamMemberRoleEnum.member]: {
+    label: 'user.team.role.member'
+  },
+  [TeamMemberRoleEnum.visitor]: {
+    label: 'user.team.role.visitor'
+  }
+};

packages/global/support/user/controller.d.ts

@@ -0,0 +1,21 @@
+export type CreateTeamProps = {
+  ownerId: string;
+  name: string;
+  avatar?: string;
+};
+export type UpdateTeamProps = {
+  id: string;
+  name?: string;
+  avatar?: string;
+};
+export type updateTeamBalanceProps = {
+  id: string;
+  balance: number;
+};
+
+export type CreateTeamMemberProps = {
+  ownerId: string;
+  teamId: string;
+  userId: string;
+  name?: string;
+};

packages/global/support/user/type.d.ts

@@ -0,0 +1,48 @@
+import { InformTypeEnum, TeamMemberRoleEnum } from './constant';
+
+export type UserModelSchema = {
+  _id: string;
+  username: string;
+  password: string;
+  avatar: string;
+  balance: number;
+  promotionRate: number;
+  inviterId?: string;
+  openaiKey: string;
+  createTime: number;
+  timezone: string;
+  openaiAccount?: {
+    key: string;
+    baseUrl: string;
+  };
+  limit: {
+    exportKbTime?: Date;
+    datasetMaxCount?: number;
+  };
+};
+
+export type UserInformSchema = {
+  _id: string;
+  userId: string;
+  time: Date;
+  type: `${InformTypeEnum}`;
+  title: string;
+  content: string;
+  read: boolean;
+};
+
+export type TeamSchema = {
+  _id: string;
+  name: string;
+  ownerId: string;
+  avatar: string;
+  createTime: Date;
+};
+
+export type TeamMemberSchema = {
+  _id: string;
+  name: string;
+  teamId: string;
+  userId: string;
+  role: `${TeamMemberRoleEnum}`;
+};

packages/global/support/wallet/type.d.ts

@@ -0,0 +1,8 @@
+export type PaySchema = {
+  _id: string;
+  userId: string;
+  createTime: Date;
+  price: number;
+  orderId: string;
+  status: 'SUCCESS' | 'REFUND' | 'NOTPAY' | 'CLOSED';
+};

packages/service/common/file/image/constant.ts

@@ -0,0 +1 @@
+export const imageBaseUrl = '/api/system/img/';

packages/service/common/file/image/controller.ts

@@ -0,0 +1,25 @@
+import { imageBaseUrl } from './constant';
+import { MongoImage } from './schema';
+
+export function getMongoImgUrl(id: string) {
+  return `${imageBaseUrl}${id}`;
+}
+
+export async function uploadMongoImg({ base64Img, userId }: { base64Img: string; userId: string }) {
+  const base64Data = base64Img.split(',')[1];
+
+  const { _id } = await MongoImage.create({
+    userId,
+    binary: Buffer.from(base64Data, 'base64')
+  });
+
+  return getMongoImgUrl(String(_id));
+}
+
+export async function readMongoImg({ id }: { id: string }) {
+  const data = await MongoImage.findById(id);
+  if (!data) {
+    return Promise.reject('Image not found');
+  }
+  return data?.binary;
+}

packages/service/common/file/image/schema.ts

@@ -1,4 +1,4 @@
-import { connectionMongo, type Model } from '@fastgpt/common/mongo';
+import { connectionMongo, type Model } from '../../mongo';
 const { Schema, model, models } = connectionMongo;
 
 const ImageSchema = new Schema({
@@ -12,5 +12,5 @@ const ImageSchema = new Schema({
   }
 });
 
-export const Image: Model<{ userId: string; binary: Buffer }> =
+export const MongoImage: Model<{ userId: string; binary: Buffer }> =
   models['image'] || model('image', ImageSchema);

packages/service/common/mongo/sessionRun.ts

@@ -0,0 +1,39 @@
+import mongoose from './index';
+
+export class MongoSession {
+  tasks: (() => Promise<any>)[] = [];
+  session: mongoose.mongo.ClientSession | null = null;
+  opts: {
+    session: mongoose.mongo.ClientSession;
+    new: boolean;
+  } | null = null;
+
+  constructor() {}
+  async init() {
+    this.session = await mongoose.startSession();
+    this.opts = { session: this.session, new: true };
+  }
+  push(
+    tasks: ((opts: {
+      session: mongoose.mongo.ClientSession;
+      new: boolean;
+    }) => () => Promise<any>)[] = []
+  ) {
+    if (!this.opts) return;
+    // this.tasks = this.tasks.concat(tasks.map((item) => item(this.opts)));
+  }
+  async run() {
+    if (!this.session || !this.opts) return;
+    try {
+      this.session.startTransaction();
+
+      const opts = { session: this.session, new: true };
+
+      await this.session.commitTransaction();
+    } catch (error) {
+      await this.session.abortTransaction();
+      console.error(error);
+    }
+    this.session.endSession();
+  }
+}