perf: prompt textarea (#625)

* Docs:fix sealos.md

* Docs:fix sealos.md

Dockerfile

@@ -1,57 +1,81 @@
-# Install dependencies only when needed
-FROM node:18.15-alpine AS deps
-# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk add --no-cache libc6-compat && npm install -g pnpm
+# --------- install dependence -----------
+FROM node:18.17-alpine AS mainDeps
 WORKDIR /app
 
 ARG name
+ARG proxy
+
+RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
+RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
+# if proxy exists, set proxy
+RUN [ -z "$proxy" ] || pnpm config set registry https://registry.npm.taobao.org
 
 # copy packages and one project
-COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY ./packages ./packages
 COPY ./projects/$name/package.json ./projects/$name/package.json
 
 RUN [ -f pnpm-lock.yaml ] || (echo "Lockfile not found." && exit 1)
 
-RUN pnpm install
+RUN pnpm i
 
-# Rebuild the source code only when needed
-FROM node:18.15-alpine AS builder
+# --------- install dependence -----------
+FROM node:18.17-alpine AS workerDeps
+WORKDIR /app
+
+ARG proxy
+
+RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
+RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
+# if proxy exists, set proxy
+RUN [ -z "$proxy" ] || pnpm config set registry https://registry.npm.taobao.org
+
+COPY ./worker /app/worker
+RUN cd /app/worker && pnpm i --production --ignore-workspace
+
+# --------- builder -----------
+FROM node:18.17-alpine AS builder
 WORKDIR /app
 
 ARG name
+ARG proxy
 
 # copy common node_modules and one project node_modules
 COPY package.json pnpm-workspace.yaml ./
-COPY --from=deps /app/node_modules ./node_modules
-COPY --from=deps /app/packages ./packages
+COPY --from=mainDeps /app/node_modules ./node_modules
+COPY --from=mainDeps /app/packages ./packages
 COPY ./projects/$name ./projects/$name
-COPY --from=deps /app/projects/$name/node_modules ./projects/$name/node_modules
+COPY --from=mainDeps /app/projects/$name/node_modules ./projects/$name/node_modules
 
-# Uncomment the following line in case you want to disable telemetry during the build.
-ENV NEXT_TELEMETRY_DISABLED 1
-RUN npm install -g pnpm
-RUN pnpm --filter=$name run build
+RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
 
-FROM node:18.15-alpine AS runner
+RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
+RUN pnpm --filter=$name build
+
+# --------- runner -----------
+FROM node:18.17-alpine AS runner
 WORKDIR /app
 
 ARG name
+ARG proxy
 
 # create user and use it
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
+RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
 RUN apk add --no-cache curl ca-certificates \
   && update-ca-certificates
 
 # copy running files
-COPY --from=builder /app/projects/$name/public ./projects/$name/public
-COPY --from=builder /app/projects/$name/next.config.js ./projects/$name/next.config.js
-COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/standalone ./
-COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/static ./projects/$name/.next/static
+COPY --from=builder /app/projects/$name/public /app/projects/$name/public
+COPY --from=builder /app/projects/$name/next.config.js /app/projects/$name/next.config.js
+COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/standalone /app/
+COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/static /app/projects/$name/.next/static
 # copy package.json to version file
 COPY --from=builder /app/projects/$name/package.json ./package.json 
+# copy woker
+COPY --from=workerDeps /app/worker /app/worker
 
 ENV NODE_ENV production
 ENV NEXT_TELEMETRY_DISABLED 1

README.md

@@ -50,19 +50,20 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 ## 💡 功能
 
-1. 强大的可视化编排，轻松构建 AI 应用
+`1` 强大的可视化编排，轻松构建 AI 应用
    - [x] 提供简易模式，无需操作编排
    - [x] 用户对话前引导，全局字符串变量
    - [x] 知识库搜索
    - [x] 多 LLM 模型对话
    - [x] 文本内容提取成结构化数据
    - [x] HTTP 扩展
-   - [ ] 嵌入 Laf，实现在线编写 HTTP 模块
+   - [ ] 嵌入 [Laf](https://github.com/labring/laf)，实现在线编写 HTTP 模块
    - [x] 对话下一步指引
    - [ ] 对话多路线选择
    - [x] 源文件引用追踪
    - [x] 模块封装，实现多级复用
-2. 丰富的知识库预处理
+
+`2` 丰富的知识库预处理
    - [x] 多库复用，混用
    - [x] chunk 记录修改和删除
    - [x] 支持手动输入，直接分段，QA 拆分导入
@@ -70,15 +71,18 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
    - [x] 支持知识库单独设置向量模型
    - [x] 源文件存储
    - [ ] 文件学习 Agent
-3. 多种效果测试渠道
+  
+`3` 多种效果测试渠道
    - [x] 知识库单点搜索测试
    - [x] 对话时反馈引用并可修改与删除
    - [x] 完整上下文呈现
    - [x] 完整模块中间值呈现
-4. OpenAPI
+  
+`4` OpenAPI
    - [x] completions 接口 (对齐 GPT 接口)
    - [ ] 知识库 CRUD
-5. 运营功能
+  
+`5` 运营功能
    - [x] 免登录分享窗口
    - [x] Iframe 一键嵌入
    - [x] 统一查阅对话记录，并对数据进行标注
@@ -93,7 +97,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 
 - **⚡ 快速部署**
 
-  > Sealos 的服务器在国外，不需要额外处理网络问题，无需服务器、无需魔法、无需域名，支持高并发 & 动态伸缩。点击以下按钮即可一键部署 👇
+  > [Sealos](https://sealos.io) 的服务器在国外，不需要额外处理网络问题，无需服务器、无需魔法、无需域名，支持高并发 & 动态伸缩。点击以下按钮即可一键部署 👇
 
   [![](https://cdn.jsdelivr.us/gh/labring-actions/templates@main/Deploy-on-Sealos.svg)](https://cloud.sealos.io/?openapp=system-fastdeploy%3FtemplateName%3Dfastgpt)
 
@@ -142,7 +146,7 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
     <img src="https://img.shields.io/badge/-返回顶部-7d09f1.svg" alt="#" align="right">
 </a>
 
-## 🤝 第三方生态
+## 🌿 第三方生态
 
 - [OnWeChat 个人微信/企微机器人](https://doc.fastgpt.in/docs/use-cases/onwechat/)
 
@@ -150,9 +154,50 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
     <img src="https://img.shields.io/badge/-返回顶部-7d09f1.svg" alt="#" align="right">
 </a>
 
+## 🤝 参与贡献
+
+我们非常欢迎各种形式的贡献。如果你对贡献代码感兴趣，可以查看我们的 GitHub [Issues](https://github.com/labring/FastGPT/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc)，大展身手，向我们展示你的奇思妙想。
+
+<a href="https://github.com/labring/FastGPT/graphs/contributors" target="_blank">
+  <table>
+    <tr>
+      <th colspan="2">
+        <br><img src="https://contrib.rocks/image?repo=labring/FastGPT"><br><br>
+      </th>
+    </tr>
+    <tr>
+      <td>
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://next.ossinsight.io/widgets/official/compose-org-active-contributors/thumbnail.png?activity=active&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=2x3&color_scheme=dark">
+          <img alt="Active participants of labring - past 28 days" src="https://next.ossinsight.io/widgets/official/compose-org-active-contributors/thumbnail.png?activity=active&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=2x3&color_scheme=light">
+        </picture>
+      </td>
+      <td rowspan="2">
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://next.ossinsight.io/widgets/official/compose-org-participants-growth/thumbnail.png?activity=new&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=4x7&color_scheme=dark">
+          <img alt="New trends of labring" src="https://next.ossinsight.io/widgets/official/compose-org-participants-growth/thumbnail.png?activity=new&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=4x7&color_scheme=light">
+        </picture>
+      </td>
+    </tr>
+    <tr>
+      <td>
+        <picture>
+          <source media="(prefers-color-scheme: dark)" srcset="https://next.ossinsight.io/widgets/official/compose-org-active-contributors/thumbnail.png?activity=new&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=2x3&color_scheme=dark">
+          <img alt="New participants of labring - past 28 days" src="https://next.ossinsight.io/widgets/official/compose-org-active-contributors/thumbnail.png?activity=new&period=past_28_days&owner_id=102226726&repo_ids=605673387&image_size=2x3&color_scheme=light">
+        </picture>
+      </td>
+    </tr>
+  </table>
+</a>
+
 ## 🌟 Star History
 
-[![Star History Chart](https://api.star-history.com/svg?repos=labring/FastGPT&type=Date)](https://star-history.com/#labring/FastGPT&Date)
+<a href="https://github.com/labring/FastGPT/stargazers" target="_blank" style="display: block" align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://next.ossinsight.io/widgets/official/analyze-repo-stars-history/thumbnail.png?repo_id=605673387&image_size=auto&color_scheme=dark">
+    <img alt="Star History of labring/FastGPT" src="https://next.ossinsight.io/widgets/official/analyze-repo-stars-history/thumbnail.png?repo_id=605673387&image_size=auto&color_scheme=light">
+  </picture>
+</a>
 
 <a href="#readme">
     <img src="https://img.shields.io/badge/-返回顶部-7d09f1.svg" alt="#" align="right">
@@ -165,4 +210,4 @@ https://github.com/labring/FastGPT/assets/15308462/7d3a38df-eb0e-4388-9250-2409b
 1. 允许作为后台服务直接商用，但不允许提供 SaaS 服务。
 2. 未经商业授权，任何形式的商用服务均需保留相关版权信息。
 3. 完整请查看 [FastGPT Open Source License](./LICENSE)
-4. 联系方式：yujinlong@sealos.io，[点击查看商业版定价策略](https://doc.fastgpt.in/docs/commercial)
+4. 联系方式：yujinlong@sealos.io，[点击查看商业版定价策略](https://doc.fastgpt.in/docs/commercial)

docSite/content/docs/course/data_search.md

@@ -0,0 +1,40 @@
+---
+title: '知识库搜索参数'
+description: '知识库搜索原理'
+icon: 'language'
+draft: false
+toc: true
+weight: 106
+---
+
+在知识库搜索的方式上，FastGPT提供了三种方式，分别为“语义检索”“增强语义检索”“混合检索”。
+
+![](/imgs/data_search1.png)
+
+## 语义检索
+
+语义检索就是向量检索，同时把用户的问题和知识库内容向量化，然后通过“语义相关度匹配”的方式从知识库中查找到匹配的知识点。
+
+优点：
+- 相近语义理解
+- 跨多语言理解（例如输入中文问题匹配英文知识点）
+- 多模态理解（文本，图片，音视频等）
+
+## 增强语义检索
+
+在语义检索的基础上，增强“语义相关度匹配”并在搜索结束后进行 Rerank（重排）。
+
+Rerank（重排）：把检索结果按“与用户问题语义”相关性，从高到低排序，简单的说就是把最匹配用户问题的检索结果排在前面。
+
+## 混合检索（推荐）
+
+
+在向量检索的同时进行全文检索，并把两项检索的结果混合一起重排，以便选中匹配用户问题的最佳结果。
+
+全文检索：理解为全文关键词检索，通过关键词查询知识库，并返回包含关键词的文本片段。
+
+优点：
+- 精确匹配（姓名，编号，ID等）
+- 少量关键词匹配（当用户问题字数过少时向量检索效果非常不好）
+
+混合检索结合了向量检索和全文检索的优点，并且对查询结果进行了重排，大大提高了命中率，推荐使用。

docSite/content/docs/course/websync.md

@@ -0,0 +1,78 @@
+---
+title: 'Web 站点同步'
+description: 'FastGPT Web 站点同步功能介绍和使用方式'
+icon: 'language'
+draft: false
+toc: true
+weight: 105
+---
+
+![](/imgs/webSync1.jpg)
+
+## 什么是 Web 站点同步
+
+Web 站点同步利用爬虫的技术，可以通过一个入口网站，自动捕获`同域名`下的所有网站，目前最多支持`200`个子页面。出于合规与安全角度，FastGPT 仅支持`静态站点`的爬取，主要用于各个文档站点快速构建知识库。
+
+Tips: 国内的媒体站点基本不可用，公众号、csdn、知乎等。可以通过终端发送`curl`请求检测是否为静态站点，例如：
+
+```bash
+curl ai.fastgpt.in
+```
+
+## 如何使用
+
+### 1. 新建知识库，选择 Web 站点同步
+
+![](/imgs/webSync2.jpg)
+
+![](/imgs/webSync3.jpg)
+
+### 2. 点击配置站点信息
+
+![](/imgs/webSync4.jpg)
+
+### 3. 填写网址和选择器
+
+![](/imgs/webSync5.jpg)
+
+好了， 现在点击开始同步，静等系统自动抓取网站信息即可。
+
+
+## 创建应用，绑定知识库
+
+![](/imgs/webSync6.jpg)
+
+## 选择器如何使用
+
+选择器是 HTML CSS JS 的产物，你可以通过选择器来定位到你需要抓取的具体内容，而不是整个站点。使用方式为：
+
+### 首先打开浏览器调试面板（通常是 F12，或者【右键 - 检查】）
+
+![](/imgs/webSync7.jpg)
+
+![](/imgs/webSync8.jpg)
+
+### 输入对应元素的选择器
+
+[菜鸟教程 css 选择器](https://www.runoob.com/cssref/css-selectors.html)，具体选择器的使用方式可以参考菜鸟教程。
+
+上图中，我们选中了一个区域，对应的是`div`标签，它有 `data-prismjs-copy`, `data-prismjs-copy-success`, `data-prismjs-copy-error` 三个属性，这里我们用到一个就够。所以选择器是：
+**`div[data-prismjs-copy]`**
+
+除了属性选择器，常见的还有类和ID选择器。例如：
+
+![](/imgs/webSync9.jpg)
+
+上图 class 里的是类名（可能包含多个类名，都是空格隔开的，选择一个即可），选择器可以为：**`.docs-content`**
+
+### 多选择器使用
+
+在开头的演示中，我们对 FastGPT 文档是使用了多选择器的方式来选择，通过逗号隔开了两个选择器。
+
+![](/imgs/webSync10.jpg)
+
+我们希望选中上图两个标签中的内容，此时就需要两组选择器。一组是：`.docs-content .mb-0.d-flex`，含义是 `docs-content` 类下同时包含 `mb-0`和`d-flex` 两个类的子元素；
+
+另一组是`.docs-content div[data-prismjs-copy]`，含义是`docs-content` 类下包含`data-prismjs-copy`属性的`div`元素。
+
+把两组选择器用逗号隔开即可：`.docs-content .mb-0.d-flex, .docs-content div[data-prismjs-copy]`

docSite/content/docs/development/one-api.md

@@ -1,16 +1,15 @@
 ---
 title: '接入微软、ChatGLM、本地模型等'
-description: '通过接入 One API 来实现对各种大模型的支持'
+description: '部署和接入 OneAPI，实现对各种大模型的支持'
 icon: 'Api'
 draft: false
 toc: true
 weight: 708
 ---
 
-* 默认情况下，FastGPT 只配置了 GPT 的 3 个模型，如果你需要接入其他模型，需要进行一些额外配置。
+* 默认情况下，FastGPT 只配置了 GPT 的模型，如果你需要接入其他模型，需要进行一些额外配置。
 * [One API](https://github.com/songquanpeng/one-api) 是一个 OpenAI 接口管理 & 分发系统，可以通过标准的 OpenAI API 格式访问所有的大模型，开箱即用。
-
-FastGPT 可以通过接入 One API 来实现对各种大模型的支持。部署方法也很简单。
+* FastGPT 可以通过接入 OneAPI 来实现对不同大模型的支持。OneAPI 的部署方法也很简单。
 
 ## MySQL 版本
 
@@ -51,7 +50,7 @@ BATCH_UPDATE_ENABLED=true
 BATCH_UPDATE_INTERVAL=60
 ```
 
-## 使用步骤
+## One API使用步骤
 
 ### 1. 登录 One API
 

docSite/content/docs/development/openapi/dataset.md

@@ -179,7 +179,7 @@ curl --location --request POST 'https://fastgpt.run/api/core/dataset/searchTest'
 {{< tab tabName="响应示例" >}}
 {{< markdownify >}}
 
-返回 top limit 结果
+返回 top k 结果， limit 为预估条数，会按每条数据 800 tokens 的长度进行预估，20条也就是返回 16000 tokens 长度的数据，最多测试 30000 tokens 的数据。
 
 ```bash
 {

docSite/content/docs/development/openapi/share.md

@@ -24,7 +24,9 @@ weight: 860
     "success": true,
     "message": "错误提示",
     "msg": "同message, 错误提示",
-    "uid": "用户唯一凭证"
+    "data": {
+        "uid": "用户唯一凭证"
+    }
 }
 ```
 
@@ -80,7 +82,9 @@ curl --location --request POST '{{host}}/shareAuth/init' \
 ```json
 {
     "success": true,
-    "uid": "username123",
+    "data": {
+        "uid": "用户唯一凭证"
+    }
 }
 ```
 
@@ -129,7 +133,9 @@ curl --location --request POST '{{host}}/shareAuth/start' \
 ```json
 {
     "success": true,
-    "uid": "username123",
+    "data": {
+        "uid": "用户唯一凭证"
+    }
 }
 ```
 

docSite/content/docs/development/sealos.md

@@ -41,4 +41,154 @@ Sealos 的服务器在国外，不需要额外处理网络问题，无需服务
 
 ## 部署架构图
 
-![](/imgs/sealos-fastgpt.webp)
+![](/imgs/sealos-fastgpt.webp)
+
+## Sealos 使用
+
+### 简介
+
+FastGPT 商业版共包含了3个应用（fastgpt, fastgpt-plus, fastgpt-admin）和2个数据库，使用多 Api Key 时候需要安装 OneAPI（一个应用和一个数据库），总计4个应用和3个数据库。
+
+![](/imgs/onSealos1.png)
+
+点击右侧的详情，可以查看对应应用的详细信息。
+
+### 如何更新/升级 FastGPT
+[升级脚本文档](https://doc.fastgpt.in/docs/development/upgrading/)先看下文档，看下需要升级哪个版本。注意，不要跨版本升级！！！！！
+
+例如，目前是4.5 版本，要升级到4.5.1，就先把镜像版本改成v4.5.1，执行一下升级脚本，等待完成后再继续升级。如果目标版本不需要执行初始化，则可以跳过。
+
+升级步骤：
+1. 打开sealos的应用管理
+2. 有3个应用 fastgpt ， fastgpt-plugin 和 fastgpt-admin
+3. 点击对应应用右边3个点，变更。或者点详情后右上角的变更。
+4. 修改镜像名栏
+![](/imgs/onsealos2.png)
+
+5. 点击变更/重启，会自动拉取最新镜像进行更新
+6. 执行对应版本的初始化脚本
+
+### 如何获取 FastGPT 访问链接
+
+打开对应的应用，点击外网访问地址。
+
+![](/imgs/onsealos3.png)
+
+### 配置自定义域名
+
+点击对应应用的变更->点击自定义域名->填写域名-> 操作域名 Cname -> 确认 -> 确认变。
+
+![](/imgs/onsealos4.png)
+
+### 如何修改配置文件
+
+打开 Sealos 的应用管理 -> 找到对应的应用 -> 变更 -> 往下拉到高级配置，里面有个配置文件 -> 新增或点击对应的配置文件可以进行编辑 -> 点击右上角确认变。
+
+![](/imgs/onsealos5.png)
+
+[配置文件参考](https://doc.fastgpt.in/docs/development/configuration/)
+
+FeConfig 参考下面（目前未做可视化）
+```
+"FeConfig": {
+    "show_emptyChat": false, // 是否展示聊天时空白的内容
+    "show_register": true, // 展示注册按键
+    "show_appStore": false, // 应用市场（暂时不可用）
+    "show_contact": false, // 联系方式（目前不可配置，直接false）
+    "show_git": false, // 展示 github
+    "show_doc": false, // 展示文档
+    "show_pay": true, // 展示支付
+    "show_openai_account": false,  // 用户可自定义 openai key
+    "show_promotion": false, // 邀请好友机制
+    "docUrl": "https://doc.fastgpt.in", // 文档基本地址
+    "systemTitle": "FastGPT", // 系统的 title
+    "googleClientVerKey": "", // 谷歌 v3 校验前端凭证
+    "isPlus": true, // 直接设置 true    
+    "oauth": { // oauth登录
+      "github": "",
+      "google": ""
+    },
+    "limit": {
+      "exportLimitMinutes": 0 // 导出间隔限制
+    },
+    "scripts": [
+    ]
+  }
+```
+
+### 修改站点名称以及 favicon
+修改应用的环境变量，增加
+
+```
+SYSTEM_NAME=FastGPT
+SYSTEM_FAVICON=/favicon.ico
+HOME_URL=/app/list
+```
+
+SYSTEM_FAVICON 可以是一个网络地址
+
+![](/imgs/onsealos6.png)
+
+### 挂载logo
+目前暂时无法 把浏览器上的logo替换。仅支持svg，待后续可视化做了后可以全部替换。
+新增一个挂载文件，文件名为：/app/projects/app/public/icon/logo.svg ，值为 svg 对应的值。
+
+![](/imgs/onsealos7.png)
+![](/imgs/onsealos8.png)
+
+### 管理后台
+
+![](/imgs/onsealos9.png)
+
+
+### 商业版镜像配置文件
+
+```
+{
+  "license": "",
+  "system": {
+    "title": "" // 系统名称
+  },
+  "censor": {
+    "BAIDU_TEXT_CENSOR_CLIENTID": "", // 百度文本安全校验
+    "BAIDU_TEXT_CENSOR_CLIENTSECRET": "" // 百度文本安全校验
+  },
+  "auth": {
+    "googleServiceVerKey": "", // 谷歌 v3 校验
+    "github": { // github oauth
+      "clientId": "",
+      "secret": ""
+    },
+    "google": { // google oauth
+      "clientId": "",
+      "secret": ""
+    },
+    "email": { // 注册邮箱配置
+      "service": "qq",
+      "user": "",
+      "pass": ""
+    },
+    "phone": { // 阿里短信配置
+      "SNED_PHONE_ACCESSKEYID": "",
+      "SNED_PHONE_ACCESSSECRET": "",
+      "SNED_PHONE_SIGNNAME": "",
+      "SNED_PHONE_TEMPLATE": ""
+    }
+  },
+  "pay": { // 微信支付配置
+    "wx": {
+      "WX_APPID": "",
+      "WX_MCHID": "",
+      "WX_V3_CODE": "",
+      "WX_NOTIFY_URL": "",
+      "WX_SERIAL_NO": "",
+      "WX_PRIVATE_KEY": ""
+    }
+  }
+}
+
+```
+
+### One API 使用
+
+[参考 OneAPI 使用步骤](/docs/development/one-api/)

docSite/content/docs/development/upgrading/464.md

@@ -0,0 +1,43 @@
+---
+title: 'V4.6.4(需要初始化)'
+description: 'FastGPT V4.6.4'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 832
+---
+
+## 1。执行初始化 API
+
+发起 1 个 HTTP 请求 ({{rootkey}} 替换成环境变量里的 `rootkey`，{{host}} 替换成自己域名)
+
+1. https://xxxxx/api/admin/initv464
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv464' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化说明：
+1. 初始化 PG 的createTime字段
+2. 初始化 Mongo 中 chat 的 feedback 字段
+
+
+## V4.6.4 功能介绍
+
+1. 重写 - 分享链接身份逻辑，采用 localID 记录用户的ID。
+2. 商业版新增 - 分享链接 SSO 方案，通过`身份鉴权`地址，仅需`3个接口`即可完全接入已有用户系统。具体参考[分享链接身份鉴权](/docs/development/openapi/share/)
+3. 新增 - 分享链接更多嵌入方式提示，更多DIY方式。
+4. 优化 - 历史记录模块。弃用旧的历史记录模块，直接在对应地方填写数值即可。
+5. 调整 - 知识库搜索模块 topk 逻辑，采用 MaxToken 计算，兼容不同长度的文本块
+6. 调整鉴权顺序，提高 apikey 的优先级，避免cookie抢占 apikey 的鉴权。
+7. 链接读取支持多选择器。参考[Web 站点同步用法](/docs/course/websync)
+8. 修复 - 分享链接图片上传鉴权问题
+9. 修复 - Mongo 连接池未释放问题。
+10. 修复 - Dataset Intro 无法更新
+11. 修复 - md 代码块问题
+12. 修复 - root 权限问题
+13. 优化 docker file  
+
+

docSite/content/docs/use-cases/wechat.md

@@ -43,7 +43,6 @@ weight: 506
 AIBOTK_KEY=微秘书 APIKEY   
 AIBOTK_SECRET=微秘书 APISECRET   
 WORK_PRO_TOKEN=你申请的企微 token   （企业微信需要填写，私人微信不需要）
-WECHATY_PUPPET_SERVICE_AUTHORITY=token-service-discovery-test.juzibot.com（企业微信需要填写，私人微信不需要）  
 ```
 
 这里最后两个变量只有部署企业微信才需要，私人微信只需要填写前两个即可。
@@ -56,7 +55,7 @@ WECHATY_PUPPET_SERVICE_AUTHORITY=token-service-discovery-test.juzibot.com（企
 
 ![](/imgs/wechat7.png)
 
-`WORK_PRO_TOKEN` [点击这里](https://tss.juzibot.com?aff=aibotk)申请 token 然后填入即可。   
+`WORK_PRO_TOKEN` [点击这里](https://tss.rpachat.com/?aff=aibotk)申请 token 然后填入即可。   
 
 `WECHATY_PUPPET_SERVICE_AUTHORITY`的值复制过去就可以。
 

docSite/content/docs/workflow/modules/custom_feedback.md

@@ -0,0 +1,35 @@
+---
+title: "自定义反馈"
+description: "自定义反馈模块介绍"
+icon: "feedback"
+draft: false
+toc: true
+weight: 354
+---
+
+该模块为临时模块，后续会针对该模块进行更全面的设计。
+
+## 特点
+
+- 可重复添加
+- 无外部输入
+- 自动执行
+
+
+|                       |                       |
+| --------------------- | --------------------- |
+| ![](/imgs/customfeedback1.png) | ![](/imgs/customfeedback2.png) |
+| ![](/imgs/customfeedback3.png) | ![](/imgs/customfeedback4.png) |
+
+
+## 介绍
+
+自定义反馈模块，可以为你的对话增加一个反馈标记，从而方便在后台更好的分析对话的数据。
+
+在调试模式下，不会记录反馈内容，而是直接提示: `自动反馈测试: 反馈内容`。
+
+在对话模式（对话、分享窗口、带 chatId 的 API 调用）时，会将反馈内容记录到对话日志中。（会延迟60s记录）
+
+## 作用
+
+自定义反馈模块的功能类似于程序开发的`埋点`，便于你观测的对话中的数据。

docSite/content/docs/workflow/modules/history.md

@@ -1,19 +0,0 @@
----
-title: "历史记录"
-description: "FastGPT 历史记录模块介绍"
-icon: "history"
-draft: false
-toc: true
-weight: 354
----
-
-# 特点
-
-- 可重复添加（防止复杂编排时线太乱，重复添加可以更美观）
-- 无外部输入
-- 流程入口
-- 自动执行
-
-每次对话时，会从数据库取最多 n 条聊天记录作为上下文。注意，不是指本轮对话最多 n 条上下文，本轮对话还包括：提示词、限定词、引用内容和问题。
-
-![](/imgs/history.png)

docSite/content/docs/workflow/modules/http.md

@@ -1,5 +1,5 @@
 ---
-title: "HTTP 模块"
+title: "新 HTTP 模块"
 description: "FastGPT HTTP 模块介绍"
 icon: "http"
 draft: false
@@ -19,86 +19,233 @@ weight: 355
 
 ## 介绍
 
-HTTP 模块会向对应的地址发送一个 POST 请求（Body 中携带 JSON 类型的参数，具体的参数可自定义），并接收一个 JSON 响应值，字段也是自定义。如上图中，我们定义了一个入参：「提取的字段」（定义的 key 为 appointment，类型为 string）和一个出参：「提取结果」（定义的 key 为 response，类型为 string）。
+HTTP 模块会向对应的地址发送一个 `POST/GET` 请求，携带部分`系统参数`及`自定义参数`，并接收一个 JSON 响应值，字段也是自定义。
 
-那么，这个请求的命令为：
+- 你还可以通过 JSON 传入自定义的请求头。
+- POST 请求中，数据会被放置在 `body` 中。
+- GET 请求中，数据会被放置在 `query` 中。
+- 在出入参数中，你都可以通过 xxx.xxx 来代表嵌套的对象。
 
-```bash
-curl --location --request POST 'https://xxxx.laf.dev/appointment-lab' \
---header 'Content-Type: application/json' \
---data-raw '{
-  "appointment":"{\"name\":\"小明\",\"time\":\"2023/08/16 15:00\",\"labname\":\"子良A323\"}"
-}'
-```
+## 参数结构
 
-响应为：
+### 系统参数说明
+
+- appId: 应用的ID
+- chatId: 当前对话的ID，测试模式下不存在。
+- responseChatItemId: 当前对话中，响应的消息ID，测试模式下不存在。
+- variables: 当前对话的全局变量。
+- data: 自定义传递的参数。
+
+### 嵌套对象使用
+
+**入参**
+
+假设我们设计了`3个`输入。
+
+- user.name (string)
+- user.age (number)
+- type (string)
+
+最终组成的对象为: 
 
 ```json
 {
-  "response": "您已经有一个预约记录了，每人仅能同时预约一个实验室:\n  姓名：小明\n  时间: 2023/08/15 15:00\n  实验室: 子良A323\n      "
+  "user": {
+    "name": "",
+    "age": ""
+  },
+  "type": ""
 }
 ```
 
+**出参**
+
+假设接口的输出结构为: 
+
+```json
+{
+  "message": "测试",
+  "data":{
+    "name": "name",
+    "age": 10
+  }
+}
+```
+
+那么，自定出参的`key`可以设置为: 
+
+- message (string)
+- data.name (string)
+- data.age (number)
+
+
+## POST 示例
+
+**自定义入参**
+
+- user.name (string)
+- user.age (number)
+- type (string)
+
+**自定义出参**
+
+- message (string)
+- data.name (string)
+- data.age (number)
+
+那么，这个模块发出的请求则是:
+
+{{< tabs tabTotal="2" >}}
+{{< tab tabName="POST 请求示例" >}}
+{{< markdownify >}}
+
+```bash
+curl --location --request POST 'http://xxxx.com' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+  "appId": "65782f7ffae5f7854ed4498b",
+  "chatId": "xxxx",
+  "responseChatItemId": "xxxx",
+  "variables": {
+    "cTime": "2023-12-18 13:45:46"
+  },
+  "data": {
+    "user": {
+      "name": "",
+      "age": ""
+    },
+    "type": ""
+  }
+}'
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="POST响应" >}}
+{{< markdownify >}}
+
+```json
+{
+  "message": "message",
+  "data": {
+    "name": "name",
+    "age": 10
+  }
+}
+```
+{{< /markdownify >}}
+{{< /tab >}}
+{{< /tabs >}}
+
+## GET 示例
+
+GET 中，不推荐使用嵌套参数，否则会出现奇怪的问题。此外，GET 请求中，FastGPT 会将参数扁平化，不会将自定义参单独抽到 data 中，同时全局变量也会扁平化，因此需要注意字段 key 是否冲突。
+
+**自定义入参**
+
+- name (string)
+- age (number)
+- type (string)
+
+**自定义出参**
+
+- message (string)
+- name (string)
+- age (number)
+
+那么，这个模块发出的请求则是:
+
+{{< tabs tabTotal="2" >}}
+{{< tab tabName="GET 请求示例" >}}
+{{< markdownify >}}
+
+```bash
+curl --location --request GET 'http://xxx.com/test?name&age&type&appId=65782f7ffae5f7854ed4498b&chatId=xxxx&responseChatItemId=xxxx&cTime=2023-12-18 13:45:46'
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="GET 响应" >}}
+{{< markdownify >}}
+
+```json
+{
+  "message": "message",
+  "data": {
+    "name": "name",
+    "age": 10
+  }
+}
+```
+{{< /markdownify >}}
+{{< /tab >}}
+{{< /tabs >}}
+
+
+## laf 对接 HTTP 示例
+
 {{% alert context="warning" %}}
 如果你不想额外部署服务，可以使用 [Laf](https://laf.dev/) 来快速开发上线接口，即写即发，无需部署。
-
-下面是在 Laf 上编写的一个请求示例：
 {{% /alert %}}
 
+下面是在 Laf 编写的 POST 请求示例：
+
 ```ts
-import cloud from '@lafjs/cloud';
-const db = cloud.database();
+import cloud from '@lafjs/cloud'
+const db = cloud.database()
+
+type RequestType = {
+  appId: string;
+  data: {
+    appointment: string;
+    action: 'post' | 'delete' | 'put' | 'get'
+  }
+}
 
 export default async function (ctx: FunctionContext) {
-  const { appointment } = ctx.body;
-  const { name, time, labname } = JSON.parse(appointment);
+  try {
+    // 从 body 中获取参数
+    const { appId, data: { appointment, action } } = ctx.body as RequestType
+
+    const parseBody = JSON.parse(appointment)
+    if (action === 'get') {
+      return await getRecord(parseBody)
+    }
+    if (action === 'post') {
+      return await createRecord(parseBody)
+    }
+    if (action === 'put') {
+      return await putRecord(parseBody)
+    }
+    if (action === 'delete') {
+      return await removeRecord(parseBody)
+    }
 
-  const missData = [];
-  if (!name) missData.push('你的姓名');
-  if (!time) missData.push('需要预约的时间');
-  if (!labname) missData.push('实验室名称');
 
-  if (missData.length > 0) {
     return {
-      response: `请提供: ${missData.join('、')}`
-    };
-  }
-
-  const { data: record } = await db
-    .collection('LabAppointment')
-    .where({
-      name,
-      status: 'unStart'
-    })
-    .getOne();
-
-  if (record) {
+      response: "异常"
+    }
+  } catch (err) {
     return {
-      response: `您已经有一个预约记录了，每人仅能同时预约一个实验室:
-  姓名：${record.name}
-  时间: ${record.time}
-  实验室: ${record.labname}
-      `
-    };
+      response: "异常"
+    }
   }
-
-  await db.collection('LabAppointment').add({
-    name,
-    time,
-    labname,
-    status: 'unStart'
-  });
-
-  return {
-    response: `预约成功。
-  姓名：${name}
-  时间: ${time}
-  实验室: ${labname}
-  `
-  };
 }
 ```
 
 ## 作用
 
-基于 HTTP 模块可以无限扩展，比如操作数据库、执行联网搜索、发送邮箱等等。如果你有有趣的案例，欢迎提交 PR 到 [编排案例](/docs/workflow/examples)
+通过 HTTP 模块你可以无限扩展，比如：
+- 操作数据库
+- 调用外部数据源
+- 执行联网搜索
+- 发送邮箱
+- ....
+
+
+## 相关示例
+
+- [谷歌搜索](/docs/workflow/examples/google_search/)
+- [实验室预约（操作数据库）](/docs/workflow/examples/lab_appointment/)

docSite/content/docs/workflow/modules/judgement.md

@@ -0,0 +1,28 @@
+---
+title: "判断器"
+description: "FastGPT 判断器模块介绍"
+icon: "input"
+draft: false
+toc: true
+weight: 362
+---
+
+## 特点
+
+- 可重复添加
+- 有外部输入
+- 触发执行
+
+![](/imgs/judgement1.png)
+
+## 功能
+
+对任意输入内容进行 True False 输出，默认情况下，当传入的内容为 false, undefined, null,0,none 时，会输出 false。
+
+也可以增加自定义规则来补充输出 false 的内容，每行代表一个匹配规则，支持正则表达式。
+
+根据上方示例图的匹配规则，当我们输入`123` `hi` `你好` 和任意手机号码时（正则匹配）同样也会输出 False 。
+
+## 作用
+
+适用场景有：让大模型做判断后输出固定内容，根据大模型回复内容判断是否触发后续模块。

docSite/content/docs/workflow/modules/string.md

@@ -0,0 +1,28 @@
+---
+title: "文本加工"
+description: "FastGPT 文本加工模块介绍"
+icon: "input"
+draft: false
+toc: true
+weight: 363
+---
+
+## 特点
+
+- 可重复添加
+- 有外部输入
+- 触发执行
+- 手动配置
+
+![](/imgs/string.png)
+
+
+## 功能
+对输入文本进行固定加工处理，入参仅支持字符串和数字格式，入参以变量形式使用在文本编辑区域。
+
+根据上方示例图的处理方式，对任何输入都会在前面拼接“我的问题是：”。
+
+
+## 作用
+
+给任意模块输入自定格式文本，或处理 AI 模块系统提示词。

package.json

@@ -5,27 +5,24 @@
   "scripts": {
     "prepare": "husky install",
     "format-code": "prettier --config \"./.prettierrc.js\" --write \"./**/src/**/*.{ts,tsx,scss}\"",
-    "format-doc": "zhlint --dir ./docSite *.md --fix"
+    "format-doc": "zhlint --dir ./docSite *.md --fix",
+    "postinstall": "sh ./scripts/postinstall.sh"
   },
   "devDependencies": {
-    "@types/multer": "^1.4.10",
     "husky": "^8.0.3",
-    "i18next": "^22.5.1",
     "lint-staged": "^13.2.1",
-    "next-i18next": "^13.3.0",
     "prettier": "^3.0.3",
-    "react-i18next": "^12.3.1",
-    "zhlint": "^0.7.1"
+    "zhlint": "^0.7.1",
+    "i18next": "^22.5.1",
+    "next-i18next": "^13.3.0",
+    "react-i18next": "^12.3.1"
   },
   "lint-staged": {
     "./**/**/*.{ts,tsx,scss}": "npm run format-code",
     "./**/**/*.md": "npm run format-doc"
   },
   "engines": {
-    "node": ">=18.0.0"
-  },
-  "dependencies": {
-    "multer": "1.4.5-lts.1",
-    "openai": "4.16.1"
+    "node": ">=18.0.0",
+    "pnpm": ">=8.6.0"
   }
 }

packages/global/common/file/tools.ts

@@ -1,8 +1,3 @@
-import axios from 'axios';
-import { UrlFetchParams, UrlFetchResponse } from './api.d';
-import { htmlToMarkdown } from '../string/markdown';
-import * as cheerio from 'cheerio';
-
 export const formatFileSize = (bytes: number): string => {
   if (bytes === 0) return '0 B';
 
@@ -12,91 +7,3 @@ export const formatFileSize = (bytes: number): string => {
 
   return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
 };
-
-export const cheerioToHtml = ({
-  fetchUrl,
-  $,
-  selector
-}: {
-  fetchUrl: string;
-  $: cheerio.CheerioAPI;
-  selector?: string;
-}) => {
-  // get origin url
-  const originUrl = new URL(fetchUrl).origin;
-
-  // remove i element
-  $('i,script').remove();
-
-  // remove empty a element
-  $('a')
-    .filter((i, el) => {
-      return $(el).text().trim() === '' && $(el).children().length === 0;
-    })
-    .remove();
-
-  // if link,img startWith /, add origin url
-  $('a').each((i, el) => {
-    const href = $(el).attr('href');
-    if (href && href.startsWith('/')) {
-      $(el).attr('href', originUrl + href);
-    }
-  });
-  $('img').each((i, el) => {
-    const src = $(el).attr('src');
-    if (src && src.startsWith('/')) {
-      $(el).attr('src', originUrl + src);
-    }
-  });
-
-  const html = $(selector || 'body')
-    .map((item, dom) => {
-      return $(dom).html();
-    })
-    .get()
-    .join('\n');
-
-  return html;
-};
-export const urlsFetch = async ({
-  urlList,
-  selector
-}: UrlFetchParams): Promise<UrlFetchResponse> => {
-  urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
-
-  const response = (
-    await Promise.all(
-      urlList.map(async (url) => {
-        try {
-          const fetchRes = await axios.get(url, {
-            timeout: 30000
-          });
-
-          const $ = cheerio.load(fetchRes.data);
-
-          const md = htmlToMarkdown(
-            cheerioToHtml({
-              fetchUrl: url,
-              $,
-              selector
-            })
-          );
-
-          return {
-            url,
-            content: md
-          };
-        } catch (error) {
-          console.log(error, 'fetch error');
-
-          return {
-            url,
-            content: ''
-          };
-        }
-      })
-    )
-  ).filter((item) => item.content);
-
-  return response;
-};

packages/global/common/string/markdown.ts

@@ -1,5 +1,4 @@
 import { simpleText } from './tools';
-import { NodeHtmlMarkdown } from 'node-html-markdown';
 
 /* Delete redundant text in markdown */
 export const simpleMarkdownText = (rawText: string) => {
@@ -27,75 +26,11 @@ export const simpleMarkdownText = (rawText: string) => {
 
   // Remove headings and code blocks front spaces
   ['####', '###', '##', '#', '```', '~~~'].forEach((item, i) => {
-    const isMarkdown = i <= 3;
     const reg = new RegExp(`\\n\\s*${item}`, 'g');
     if (reg.test(rawText)) {
-      rawText = rawText.replace(
-        new RegExp(`(\\n)\\s*(${item})`, 'g'),
-        isMarkdown ? '\n$1$2' : '$1$2'
-      );
+      rawText = rawText.replace(new RegExp(`(\\n)\\s*(${item})`, 'g'), '$1$2');
     }
   });
 
   return rawText.trim();
 };
-
-/* html string to markdown */
-export const htmlToMarkdown = (html?: string | null) => {
-  if (!html) return '';
-
-  const surround = (source: string, surroundStr: string) => `${surroundStr}${source}${surroundStr}`;
-
-  const nhm = new NodeHtmlMarkdown(
-    {
-      codeFence: '```',
-      codeBlockStyle: 'fenced',
-      ignore: ['i', 'script']
-    },
-    {
-      code: ({ node, parent, options: { codeFence, codeBlockStyle }, visitor }) => {
-        const isCodeBlock = ['PRE', 'WRAPPED-PRE'].includes(parent?.tagName!);
-
-        if (!isCodeBlock) {
-          return {
-            spaceIfRepeatingChar: true,
-            noEscape: true,
-            postprocess: ({ content }) => {
-              // Find longest occurring sequence of running backticks and add one more (so content is escaped)
-              const delimiter =
-                '`' + (content.match(/`+/g)?.sort((a, b) => b.length - a.length)?.[0] || '');
-              const padding = delimiter.length > 1 ? ' ' : '';
-
-              return surround(surround(content, padding), delimiter);
-            }
-          };
-        }
-
-        /* Handle code block */
-        if (codeBlockStyle === 'fenced') {
-          const language =
-            node.getAttribute('class')?.match(/language-(\S+)/)?.[1] ||
-            parent?.getAttribute('class')?.match(/language-(\S+)/)?.[1] ||
-            '';
-
-          return {
-            noEscape: true,
-            prefix: `${codeFence}${language}\n`,
-            postfix: `\n${codeFence}\n`,
-            childTranslators: visitor.instance.codeBlockTranslators
-          };
-        }
-
-        return {
-          noEscape: true,
-          postprocess: ({ content }) => content.replace(/^/gm, '    '),
-          childTranslators: visitor.instance.codeBlockTranslators
-        };
-      }
-    }
-  );
-
-  const markdown = nhm.translate(html).trim();
-
-  return simpleMarkdownText(markdown);
-};

packages/global/common/string/textSplitter.ts

@@ -13,12 +13,13 @@ export const splitText2Chunks = (props: {
   chunkLen: number;
   overlapRatio?: number;
   customReg?: string[];
+  countTokens?: boolean;
 }): {
   chunks: string[];
   tokens: number;
   overlapRatio?: number;
 } => {
-  let { text = '', chunkLen, overlapRatio = 0.2, customReg = [] } = props;
+  let { text = '', chunkLen, overlapRatio = 0.2, customReg = [], countTokens = true } = props;
   const splitMarker = 'SPLIT_HERE_SPLIT_HERE';
   const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER';
   const overlapLen = Math.round(chunkLen * overlapRatio);
@@ -231,9 +232,11 @@ export const splitText2Chunks = (props: {
       step: 0,
       lastText: '',
       mdTitle: ''
-    }).map((chunk) => chunk.replaceAll(codeBlockMarker, '\n')); // restore code block
+    }).map((chunk) => chunk?.replaceAll(codeBlockMarker, '\n') || ''); // restore code block
 
-    const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0);
+    const tokens = countTokens
+      ? chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0)
+      : 0;
 
     return {
       chunks,

packages/global/common/system/constants.ts

@@ -0,0 +1,2 @@
+export const HUMAN_ICON = `/icon/human.svg`;
+export const LOGO_ICON = `/icon/logo.svg`;

packages/global/common/system/types/index.d.ts

@@ -9,6 +9,7 @@ export type FeConfigsType = {
   hide_app_flow?: boolean;
   concatMd?: string;
   docUrl?: string;
+  chatbotUrl?: string;
   openAPIDocUrl?: string;
   systemTitle?: string;
   googleClientVerKey?: string;

packages/global/core/chat/api.d.ts

@@ -0,0 +1,9 @@
+export type UpdateChatFeedbackProps = {
+  appId: string;
+  chatId: string;
+  chatItemId: string;
+  shareId?: string;
+  outLinkUid?: string;
+  userBadFeedback?: string;
+  userGoodFeedback?: string;
+};

packages/global/core/chat/constants.ts

@@ -50,9 +50,6 @@ export enum ChatStatusEnum {
   finish = 'finish'
 }
 
-export const HUMAN_ICON = `/icon/human.svg`;
-export const LOGO_ICON = `/icon/logo.svg`;
-
 export const IMG_BLOCK_KEY = 'img-block';
 export const FILE_BLOCK_KEY = 'file-block';
 

packages/global/core/chat/type.d.ts

@@ -38,7 +38,9 @@ export type ChatItemSchema = {
   time: Date;
   obj: `${ChatRoleEnum}`;
   value: string;
-  userFeedback?: string;
+  userGoodFeedback?: string;
+  userBadFeedback?: string;
+  customFeedbacks?: string[];
   adminFeedback?: AdminFbkType;
   [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };
@@ -56,7 +58,9 @@ export type ChatItemType = {
   dataId?: string;
   obj: ChatItemSchema['obj'];
   value: any;
-  userFeedback?: string;
+  userGoodFeedback?: string;
+  userBadFeedback?: string;
+  customFeedbacks?: ChatItemSchema['customFeedbacks'];
   adminFeedback?: ChatItemSchema['feedback'];
   [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };
@@ -81,12 +85,14 @@ export type ChatHistoryItemType = HistoryItemType & {
 
 /* ------- response data ------------ */
 export type moduleDispatchResType = {
+  // common
   moduleLogo?: string;
   price?: number;
   runningTime?: number;
   tokens?: number;
   model?: string;
   query?: string;
+  contextTotalLen?: number;
 
   // chat
   temperature?: number;
@@ -113,6 +119,12 @@ export type moduleDispatchResType = {
 
   // plugin output
   pluginOutput?: Record<string, any>;
+
+  // text editor
+  textOutput?: string;
+
+  // tf switch
+  tfSwitchResult?: boolean;
 };
 
 export type ChatHistoryItemResType = moduleDispatchResType & {

packages/global/core/dataset/api.d.ts

@@ -25,6 +25,8 @@ export type CreateDatasetCollectionParams = {
   chunkSize?: number;
   fileId?: string;
   rawLink?: string;
+  qaPrompt?: string;
+  hashRawText?: string;
   metadata?: Record<string, any>;
 };
 
@@ -40,6 +42,7 @@ export type PgSearchRawType = {
 export type PushDatasetDataChunkProps = {
   q: string; // embedding content
   a?: string; // bonus content
+  chunkIndex?: number;
   indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
 };
 

packages/global/core/dataset/type.d.ts

@@ -45,6 +45,8 @@ export type DatasetCollectionSchemaType = {
   chunkSize: number;
   fileId?: string;
   rawLink?: string;
+  qaPrompt?: string;
+  hashRawText?: string;
   metadata?: Record<string, any>;
 };
 
@@ -134,6 +136,7 @@ export type DatasetDataItemType = {
   sourceId?: string;
   q: string;
   a: string;
+  chunkIndex: number;
   indexes: DatasetDataIndexItemType[];
   isOwner: boolean;
   canWrite: boolean;

packages/global/core/module/api.d.ts

@@ -1,3 +1,18 @@
 import { VectorModelItemType } from '../ai/model.d';
 
 export type SelectedDatasetType = { datasetId: string; vectorModel: VectorModelItemType }[];
+
+export type HttpBodyType<T = any> = {
+  appId: string;
+  chatId?: string;
+  responseChatItemId?: string;
+  variables: Record<string, any>;
+  data: T;
+};
+export type HttpQueryType = {
+  appId: string;
+  chatId?: string;
+  responseChatItemId?: string;
+  variables: Record<string, any>;
+  [key: string]: any;
+};

packages/global/core/module/constants.ts

@@ -1,19 +1,17 @@
 export enum ModuleTemplateTypeEnum {
   userGuide = 'userGuide',
   systemInput = 'systemInput',
+  tools = 'tools',
   textAnswer = 'textAnswer',
-  dataset = 'dataset',
   functionCall = 'functionCall',
   externalCall = 'externalCall',
 
   personalPlugin = 'personalPlugin',
-  communityPlugin = 'communityPlugin',
-  commercialPlugin = 'commercialPlugin',
 
   other = 'other'
 }
 
-export enum ModuleDataTypeEnum {
+export enum ModuleIOValueTypeEnum {
   string = 'string',
   number = 'number',
   boolean = 'boolean',
@@ -44,6 +42,9 @@ export enum ModuleInputKeyEnum {
   aiModel = 'model',
   aiSystemPrompt = 'systemPrompt',
   description = 'description',
+  anyInput = 'system_anyInput',
+  textareaInput = 'system_textareaInput',
+  addInputParam = 'system_addInputParam',
 
   // history
   historyMaxAmount = 'maxContext',
@@ -69,7 +70,10 @@ export enum ModuleInputKeyEnum {
   extractKeys = 'extractKeys',
 
   // http
-  httpUrl = 'url',
+  httpReqUrl = 'system_httpReqUrl',
+  httpHeader = 'system_httpHeader',
+  httpMethod = 'system_httpMethod',
+  abandon_httpUrl = 'url',
 
   // app
   runAppSelectApp = 'app',
@@ -87,6 +91,8 @@ export enum ModuleOutputKeyEnum {
   answerText = 'answerText', //  answer module text key
   success = 'success',
   failed = 'failed',
+  text = 'system_text',
+  addOutputParam = 'system_addOutputParam',
 
   // dataset
   datasetIsEmpty = 'isEmpty',
@@ -94,7 +100,11 @@ export enum ModuleOutputKeyEnum {
   datasetQuoteQA = 'quoteQA',
 
   // context extract
-  contextExtractFields = 'fields'
+  contextExtractFields = 'fields',
+
+  // tf switch
+  resultTrue = 'system_resultTrue',
+  resultFalse = 'system_resultFalse'
 }
 
 export enum VariableInputEnum {
@@ -102,3 +112,5 @@ export enum VariableInputEnum {
   textarea = 'textarea',
   select = 'select'
 }
+
+export const DYNAMIC_INPUT_KEY = 'DYNAMIC_INPUT_KEY';

packages/global/core/module/node/constant.ts

@@ -2,29 +2,39 @@ export enum FlowNodeInputTypeEnum {
   systemInput = 'systemInput', // history, userChatInput, variableInput
 
   input = 'input', // one line input
-  textarea = 'textarea',
   numberInput = 'numberInput',
   select = 'select',
   slider = 'slider',
-  custom = 'custom',
   target = 'target', // data input
   switch = 'switch',
+  textarea = 'textarea',
+
+  addInputParam = 'addInputParam', // params input
+
   selectApp = 'selectApp',
+
   // chat special input
   aiSettings = 'aiSettings',
-  // maxToken = 'maxToken',
+
+  // ai model select
   selectChatModel = 'selectChatModel',
+  selectCQModel = 'selectCQModel',
+  selectExtractModel = 'selectExtractModel',
+
   // dataset special input
   selectDataset = 'selectDataset',
   selectDatasetParamsModal = 'selectDatasetParamsModal',
 
-  hidden = 'hidden'
+  hidden = 'hidden',
+  custom = 'custom'
 }
 
 export enum FlowNodeOutputTypeEnum {
   answer = 'answer',
   source = 'source',
-  hidden = 'hidden'
+  hidden = 'hidden',
+
+  addOutputParam = 'addOutputParam'
 }
 
 export enum FlowNodeTypeEnum {
@@ -42,7 +52,10 @@ export enum FlowNodeTypeEnum {
   pluginModule = 'pluginModule',
   pluginInput = 'pluginInput',
   pluginOutput = 'pluginOutput',
+  textEditor = 'textEditor',
 
   // abandon
   variable = 'variable'
 }
+
+export const EDGE_TYPE = 'smoothstep';

packages/global/core/module/node/type.d.ts

@@ -1,6 +1,7 @@
 import { FlowNodeInputTypeEnum, FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from './constant';
-import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleOutputKeyEnum } from '../constants';
+import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, ModuleOutputKeyEnum } from '../constants';
 import { SelectedDatasetType } from '../api';
+import { EditInputFieldMap, EditOutputFieldMap } from './type';
 
 export type FlowNodeChangeProps = {
   moduleId: string;
@@ -20,27 +21,34 @@ export type FlowNodeChangeProps = {
 };
 
 export type FlowNodeInputItemType = {
+  valueType?: `${ModuleIOValueTypeEnum}`; // data type
+  type: `${FlowNodeInputTypeEnum}`; // Node Type. Decide on a render style
   key: `${ModuleInputKeyEnum}` | string;
-  type: `${FlowNodeInputTypeEnum}`; // Decide on a render style
   value?: any;
-  valueType?: `${ModuleDataTypeEnum}`; // data type
   label: string;
   description?: string;
   required?: boolean;
+
   edit?: boolean; // Whether to allow editing
-  connected?: boolean; // unConnected field will be deleted
+  editField?: EditInputFieldMap;
+  defaultEditField?: EditNodeFieldType;
+
+  connected?: boolean; // There are incoming data
 
   showTargetInApp?: boolean;
   showTargetInPlugin?: boolean;
 
-  placeholder?: string; // input,textarea
-  list?: { label: string; value: any }[]; // select
-  step?: number; // slider max?: number;
-  max?: number;
-  min?: number;
-  markList?: { label: string; value: any }[]; // slider
+  hideInApp?: boolean;
+  hideInPlugin?: boolean;
 
-  plusField?: boolean; // plus system will show
+  placeholder?: string; // input,textarea
+
+  list?: { label: string; value: any }[]; // select
+
+  markList?: { label: string; value: any }[]; // slider
+  step?: number; // slider
+  max?: number; // slider, number input
+  min?: number; // slider, number input
 };
 
 export type FlowNodeOutputTargetItemType = {
@@ -48,15 +56,41 @@ export type FlowNodeOutputTargetItemType = {
   key: string;
 };
 export type FlowNodeOutputItemType = {
-  key: `${ModuleOutputKeyEnum}` | string;
-  label?: string;
-  edit?: boolean;
-  description?: string;
-  valueType?: `${ModuleDataTypeEnum}`;
   type?: `${FlowNodeOutputTypeEnum}`;
+  key: `${ModuleOutputKeyEnum}` | string;
+  valueType?: `${ModuleIOValueTypeEnum}`;
+
+  label?: string;
+  description?: string;
+
+  edit?: boolean;
+  editField?: EditOutputFieldMap;
+  defaultEditField?: EditNodeFieldType;
+
   targets: FlowNodeOutputTargetItemType[];
 };
 
+/* --------------- edit field ------------------- */
+export type EditInputFieldMap = EditOutputFieldMap & {
+  inputType?: boolean;
+  required?: boolean;
+};
+export type EditOutputFieldMap = {
+  name?: boolean;
+  key?: boolean;
+  description?: boolean;
+  dataType?: boolean;
+};
+export type EditNodeFieldType = {
+  inputType?: `${FlowNodeInputTypeEnum}`; // input type
+  outputType?: `${FlowNodeOutputTypeEnum}`;
+  required?: boolean;
+  key?: string;
+  label?: string;
+  description?: string;
+  valueType?: `${ModuleIOValueTypeEnum}`;
+};
+
 /* ------------- item type --------------- */
 /* ai chat modules props */
 export type AIChatModuleProps = {

packages/global/core/module/template/input.ts

@@ -1,22 +1,26 @@
 import type { FlowNodeInputItemType } from '../node/type.d';
-import { ModuleInputKeyEnum } from '../constants';
+import { DYNAMIC_INPUT_KEY, ModuleInputKeyEnum } from '../constants';
 import { FlowNodeInputTypeEnum } from '../node/constant';
-import { ModuleDataTypeEnum } from '../constants';
+import { ModuleIOValueTypeEnum } from '../constants';
 
-export const Input_Template_TFSwitch: FlowNodeInputItemType = {
+export const Input_Template_Switch: FlowNodeInputItemType = {
   key: ModuleInputKeyEnum.switch,
   type: FlowNodeInputTypeEnum.target,
   label: 'core.module.input.label.switch',
-  valueType: ModuleDataTypeEnum.any,
+  valueType: ModuleIOValueTypeEnum.any,
   showTargetInApp: true,
   showTargetInPlugin: true
 };
 
 export const Input_Template_History: FlowNodeInputItemType = {
   key: ModuleInputKeyEnum.history,
-  type: FlowNodeInputTypeEnum.target,
+  type: FlowNodeInputTypeEnum.numberInput,
   label: 'core.module.input.label.chat history',
-  valueType: ModuleDataTypeEnum.chatHistory,
+  required: true,
+  min: 0,
+  max: 30,
+  valueType: ModuleIOValueTypeEnum.chatHistory,
+  value: 6,
   showTargetInApp: true,
   showTargetInPlugin: true
 };
@@ -26,7 +30,29 @@ export const Input_Template_UserChatInput: FlowNodeInputItemType = {
   type: FlowNodeInputTypeEnum.target,
   label: 'core.module.input.label.user question',
   required: true,
-  valueType: ModuleDataTypeEnum.string,
+  valueType: ModuleIOValueTypeEnum.string,
   showTargetInApp: true,
   showTargetInPlugin: true
 };
+
+export const Input_Template_AddInputParam: FlowNodeInputItemType = {
+  key: ModuleInputKeyEnum.addInputParam,
+  type: FlowNodeInputTypeEnum.addInputParam,
+  valueType: ModuleIOValueTypeEnum.any,
+  label: '',
+  required: false,
+  showTargetInApp: false,
+  showTargetInPlugin: false
+};
+
+export const Input_Template_DynamicInput: FlowNodeInputItemType = {
+  key: DYNAMIC_INPUT_KEY,
+  type: FlowNodeInputTypeEnum.target,
+  valueType: ModuleIOValueTypeEnum.any,
+  label: 'core.module.inputType.dynamicTargetInput',
+  description: 'core.module.input.description.dynamic input',
+  required: false,
+  showTargetInApp: false,
+  showTargetInPlugin: true,
+  hideInApp: true
+};

packages/global/core/module/template/output.ts

@@ -1,13 +1,21 @@
 import type { FlowNodeOutputItemType } from '../node/type';
 import { ModuleOutputKeyEnum } from '../constants';
 import { FlowNodeOutputTypeEnum } from '../node/constant';
-import { ModuleDataTypeEnum } from '../constants';
+import { ModuleIOValueTypeEnum } from '../constants';
 
 export const Output_Template_Finish: FlowNodeOutputItemType = {
   key: ModuleOutputKeyEnum.finish,
   label: 'core.module.output.label.running done',
   description: 'core.module.output.description.running done',
-  valueType: ModuleDataTypeEnum.boolean,
+  valueType: ModuleIOValueTypeEnum.boolean,
   type: FlowNodeOutputTypeEnum.source,
   targets: []
 };
+
+export const Output_Template_AddOutput: FlowNodeOutputItemType = {
+  key: ModuleOutputKeyEnum.addOutputParam,
+  type: FlowNodeOutputTypeEnum.addOutputParam,
+  valueType: ModuleIOValueTypeEnum.any,
+  label: '',
+  targets: []
+};

packages/global/core/module/template/system/abandon/history.ts

@@ -2,17 +2,21 @@ import {
   FlowNodeInputTypeEnum,
   FlowNodeOutputTypeEnum,
   FlowNodeTypeEnum
-} from '../../node/constant';
-import { FlowModuleTemplateType } from '../../type.d';
-import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+} from '../../../node/constant';
+import { FlowModuleTemplateType } from '../../../type';
+import {
+  ModuleIOValueTypeEnum,
+  ModuleInputKeyEnum,
+  ModuleTemplateTypeEnum
+} from '../../../constants';
 
 export const HistoryModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.historyNode,
   templateType: ModuleTemplateTypeEnum.systemInput,
   flowType: FlowNodeTypeEnum.historyNode,
   avatar: '/imgs/module/history.png',
-  name: '聊天记录',
-  intro: '用户输入的内容。该模块通常作为应用的入口，用户在发送消息后会首先执行该模块。',
+  name: '聊天记录(弃用)',
+  intro: '聊天记录，该模块已被弃用',
   inputs: [
     {
       key: ModuleInputKeyEnum.historyMaxAmount,
@@ -21,7 +25,7 @@ export const HistoryModule: FlowModuleTemplateType = {
       description:
         '该记录数不代表模型可接收这么多的历史记录，具体可接收多少历史记录，取决于模型的能力，通常建议不要超过20条。',
       value: 6,
-      valueType: ModuleDataTypeEnum.number,
+      valueType: ModuleIOValueTypeEnum.number,
       min: 0,
       max: 100,
       showTargetInApp: false,
@@ -30,7 +34,7 @@ export const HistoryModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.history,
       type: FlowNodeInputTypeEnum.hidden,
-      valueType: ModuleDataTypeEnum.chatHistory,
+      valueType: ModuleIOValueTypeEnum.chatHistory,
       label: '聊天记录',
       showTargetInApp: false,
       showTargetInPlugin: false
@@ -40,7 +44,7 @@ export const HistoryModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.history,
       label: '聊天记录',
-      valueType: ModuleDataTypeEnum.chatHistory,
+      valueType: ModuleIOValueTypeEnum.chatHistory,
       type: FlowNodeOutputTypeEnum.source,
       targets: []
     }

packages/global/core/module/template/system/aiChat.ts

@@ -5,14 +5,14 @@ import {
 } from '../../node/constant';
 import { FlowModuleTemplateType } from '../../type.d';
 import {
-  ModuleDataTypeEnum,
+  ModuleIOValueTypeEnum,
   ModuleInputKeyEnum,
   ModuleOutputKeyEnum,
   ModuleTemplateTypeEnum
 } from '../../constants';
 import {
   Input_Template_History,
-  Input_Template_TFSwitch,
+  Input_Template_Switch,
   Input_Template_UserChatInput
 } from '../input';
 import { chatNodeSystemPromptTip } from '../tip';
@@ -27,13 +27,13 @@ export const AiChatModule: FlowModuleTemplateType = {
   intro: 'AI 大模型对话',
   showStatus: true,
   inputs: [
-    Input_Template_TFSwitch,
+    Input_Template_Switch,
     {
       key: ModuleInputKeyEnum.aiModel,
       type: FlowNodeInputTypeEnum.selectChatModel,
       label: '对话模型',
       required: true,
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -43,7 +43,7 @@ export const AiChatModule: FlowModuleTemplateType = {
       type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window
       label: '温度',
       value: 0,
-      valueType: ModuleDataTypeEnum.number,
+      valueType: ModuleIOValueTypeEnum.number,
       min: 0,
       max: 10,
       step: 1,
@@ -59,7 +59,7 @@ export const AiChatModule: FlowModuleTemplateType = {
       type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window
       label: '回复上限',
       value: 2000,
-      valueType: ModuleDataTypeEnum.number,
+      valueType: ModuleIOValueTypeEnum.number,
       min: 100,
       max: 4000,
       step: 50,
@@ -78,7 +78,7 @@ export const AiChatModule: FlowModuleTemplateType = {
       type: FlowNodeInputTypeEnum.hidden,
       label: '返回AI内容',
       value: true,
-      valueType: ModuleDataTypeEnum.boolean,
+      valueType: ModuleIOValueTypeEnum.boolean,
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -86,8 +86,7 @@ export const AiChatModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.aiChatQuoteTemplate,
       type: FlowNodeInputTypeEnum.hidden,
       label: '引用内容模板',
-      valueType: ModuleDataTypeEnum.string,
-      value: '',
+      valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -95,8 +94,7 @@ export const AiChatModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.aiChatQuotePrompt,
       type: FlowNodeInputTypeEnum.hidden,
       label: '引用内容提示词',
-      valueType: ModuleDataTypeEnum.string,
-      value: '',
+      valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -104,8 +102,7 @@ export const AiChatModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.aiChatSettingModal,
       type: FlowNodeInputTypeEnum.aiSettings,
       label: '',
-      connected: false,
-      valueType: ModuleDataTypeEnum.any,
+      valueType: ModuleIOValueTypeEnum.any,
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -115,24 +112,22 @@ export const AiChatModule: FlowModuleTemplateType = {
       type: FlowNodeInputTypeEnum.textarea,
       label: '系统提示词',
       max: 300,
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       description: chatNodeSystemPromptTip,
       placeholder: chatNodeSystemPromptTip,
-      value: '',
       showTargetInApp: true,
       showTargetInPlugin: true
     },
+    Input_Template_History,
     {
       key: ModuleInputKeyEnum.aiChatDatasetQuote,
       type: FlowNodeInputTypeEnum.target,
       label: '引用内容',
       description: "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
-      valueType: ModuleDataTypeEnum.datasetQuote,
-      connected: false,
+      valueType: ModuleIOValueTypeEnum.datasetQuote,
       showTargetInApp: true,
       showTargetInPlugin: true
     },
-    Input_Template_History,
     Input_Template_UserChatInput
   ],
   outputs: [
@@ -140,7 +135,7 @@ export const AiChatModule: FlowModuleTemplateType = {
       key: ModuleOutputKeyEnum.history,
       label: '新的上下文',
       description: '将本次回复内容拼接上历史记录，作为新的上下文返回',
-      valueType: ModuleDataTypeEnum.chatHistory,
+      valueType: ModuleIOValueTypeEnum.chatHistory,
       type: FlowNodeOutputTypeEnum.source,
       targets: []
     },
@@ -148,7 +143,7 @@ export const AiChatModule: FlowModuleTemplateType = {
       key: ModuleOutputKeyEnum.answerText,
       label: 'AI回复',
       description: '将在 stream 回复完毕后触发',
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       type: FlowNodeOutputTypeEnum.source,
       targets: []
     },

packages/global/core/module/template/system/assignedAnswer.ts

@@ -1,7 +1,7 @@
 import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
 import { FlowModuleTemplateType } from '../../type.d';
-import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
-import { Input_Template_TFSwitch } from '../input';
+import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+import { Input_Template_Switch } from '../input';
 import { Output_Template_Finish } from '../output';
 
 export const AssignedAnswerModule: FlowModuleTemplateType = {
@@ -12,15 +12,16 @@ export const AssignedAnswerModule: FlowModuleTemplateType = {
   name: '指定回复',
   intro: '该模块可以直接回复一段指定的内容。常用于引导、提示',
   inputs: [
-    Input_Template_TFSwitch,
+    Input_Template_Switch,
     {
       key: ModuleInputKeyEnum.answerText,
       type: FlowNodeInputTypeEnum.textarea,
-      valueType: ModuleDataTypeEnum.any,
-      value: '',
+      valueType: ModuleIOValueTypeEnum.any,
       label: '回复的内容',
       description:
-        '可以使用 \\n 来实现连续换行。\n\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n\n如传入非字符串类型数据将会自动转成字符串',
+        '可以使用 \\n 来实现连续换行。\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n如传入非字符串类型数据将会自动转成字符串',
+      placeholder:
+        '可以使用 \\n 来实现连续换行。\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n如传入非字符串类型数据将会自动转成字符串',
       showTargetInApp: true,
       showTargetInPlugin: true
     }

packages/global/core/module/template/system/classifyQuestion.ts

@@ -4,10 +4,10 @@ import {
   FlowNodeTypeEnum
 } from '../../node/constant';
 import { FlowModuleTemplateType } from '../../type.d';
-import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
 import {
   Input_Template_History,
-  Input_Template_TFSwitch,
+  Input_Template_Switch,
   Input_Template_UserChatInput
 } from '../input';
 
@@ -24,11 +24,11 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
 类型4: 其他问题`,
   showStatus: true,
   inputs: [
-    Input_Template_TFSwitch,
+    Input_Template_Switch,
     {
       key: ModuleInputKeyEnum.aiModel,
-      type: FlowNodeInputTypeEnum.selectChatModel,
-      valueType: ModuleDataTypeEnum.string,
+      type: FlowNodeInputTypeEnum.selectCQModel,
+      valueType: ModuleIOValueTypeEnum.string,
       label: '分类模型',
       required: true,
       showTargetInApp: false,
@@ -37,8 +37,7 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.aiSystemPrompt,
       type: FlowNodeInputTypeEnum.textarea,
-      valueType: ModuleDataTypeEnum.string,
-      value: '',
+      valueType: ModuleIOValueTypeEnum.string,
       label: '背景知识',
       description:
         '你可以添加一些特定内容的介绍，从而更好的识别用户的问题类型。这个内容通常是给模型介绍一个它不知道的内容。',
@@ -52,20 +51,20 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.agents,
       type: FlowNodeInputTypeEnum.custom,
-      valueType: ModuleDataTypeEnum.any,
+      valueType: ModuleIOValueTypeEnum.any,
       label: '',
       value: [
         {
           value: '打招呼',
-          key: 'fasw'
+          key: 'wqre'
         },
         {
           value: '关于 xxx 的问题',
-          key: 'fqsw'
+          key: 'sdfa'
         },
         {
           value: '其他问题',
-          key: 'fesw'
+          key: 'agex'
         }
       ],
       showTargetInApp: false,
@@ -75,19 +74,19 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
   outputs: [
     // custom output
     {
-      key: 'fasw',
+      key: 'wqre',
       label: '',
       type: FlowNodeOutputTypeEnum.hidden,
       targets: []
     },
     {
-      key: 'fqsw',
+      key: 'sdfa',
       label: '',
       type: FlowNodeOutputTypeEnum.hidden,
       targets: []
     },
     {
-      key: 'fesw',
+      key: 'agex',
       label: '',
       type: FlowNodeOutputTypeEnum.hidden,
       targets: []

packages/global/core/module/template/system/contextExtract.ts

@@ -5,12 +5,12 @@ import {
 } from '../../node/constant';
 import { FlowModuleTemplateType } from '../../type.d';
 import {
-  ModuleDataTypeEnum,
+  ModuleIOValueTypeEnum,
   ModuleInputKeyEnum,
   ModuleOutputKeyEnum,
   ModuleTemplateTypeEnum
 } from '../../constants';
-import { Input_Template_History, Input_Template_TFSwitch } from '../input';
+import { Input_Template_History, Input_Template_Switch } from '../input';
 
 export const ContextExtractModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.contentExtract,
@@ -21,17 +21,25 @@ export const ContextExtractModule: FlowModuleTemplateType = {
   intro: '可从文本中提取指定的数据，例如：sql语句、搜索关键词、代码等',
   showStatus: true,
   inputs: [
-    Input_Template_TFSwitch,
+    Input_Template_Switch,
+    {
+      key: ModuleInputKeyEnum.aiModel,
+      type: FlowNodeInputTypeEnum.selectExtractModel,
+      valueType: ModuleIOValueTypeEnum.string,
+      label: '提取模型',
+      required: true,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
     {
       key: ModuleInputKeyEnum.description,
       type: FlowNodeInputTypeEnum.textarea,
-      valueType: ModuleDataTypeEnum.string,
-      value: '',
+      valueType: ModuleIOValueTypeEnum.string,
       label: '提取要求描述',
-      description: '写一段提取要求，告诉 AI 需要提取哪些内容',
+      description: '给AI一些对应的背景知识或要求描述，引导AI更好的完成任务',
       required: true,
       placeholder:
-        '例如: \n1. 你是一个实验室预约助手。根据用户问题，提取出姓名、实验室号和预约时间',
+        '例如: \n1. 你是一个实验室预约助手，你的任务是帮助用户预约实验室。\n2. 你是谷歌搜索助手，需要从文本中提取出合适的搜索词。',
       showTargetInApp: true,
       showTargetInPlugin: true
     },
@@ -41,7 +49,7 @@ export const ContextExtractModule: FlowModuleTemplateType = {
       type: FlowNodeInputTypeEnum.target,
       label: '需要提取的文本',
       required: true,
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: true,
       showTargetInPlugin: true
     },
@@ -49,9 +57,9 @@ export const ContextExtractModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.extractKeys,
       type: FlowNodeInputTypeEnum.custom,
       label: '目标字段',
-      valueType: ModuleDataTypeEnum.any,
+      valueType: ModuleIOValueTypeEnum.any,
       description: "由 '描述' 和 'key' 组成一个目标字段，可提取多个目标字段",
-      value: [], // {desc: string; key: string; required: boolean;}[]
+      value: [], // {desc: string; key: string; required: boolean; enum: string[]}[]
       showTargetInApp: false,
       showTargetInPlugin: false
     }
@@ -60,14 +68,14 @@ export const ContextExtractModule: FlowModuleTemplateType = {
     {
       key: ModuleOutputKeyEnum.success,
       label: '字段完全提取',
-      valueType: ModuleDataTypeEnum.boolean,
+      valueType: ModuleIOValueTypeEnum.boolean,
       type: FlowNodeOutputTypeEnum.source,
       targets: []
     },
     {
       key: ModuleOutputKeyEnum.failed,
       label: '提取字段缺失',
-      valueType: ModuleDataTypeEnum.boolean,
+      valueType: ModuleIOValueTypeEnum.boolean,
       type: FlowNodeOutputTypeEnum.source,
       targets: []
     },
@@ -75,7 +83,7 @@ export const ContextExtractModule: FlowModuleTemplateType = {
       key: ModuleOutputKeyEnum.contextExtractFields,
       label: '完整提取结果',
       description: '一个 JSON 字符串，例如：{"name:":"YY","Time":"2023/7/2 18:00"}',
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       type: FlowNodeOutputTypeEnum.source,
       targets: []
     }

packages/global/core/module/template/system/datasetSearch.ts

@@ -5,31 +5,31 @@ import {
 } from '../../node/constant';
 import { FlowModuleTemplateType } from '../../type.d';
 import {
-  ModuleDataTypeEnum,
+  ModuleIOValueTypeEnum,
   ModuleInputKeyEnum,
   ModuleOutputKeyEnum,
   ModuleTemplateTypeEnum
 } from '../../constants';
-import { Input_Template_TFSwitch, Input_Template_UserChatInput } from '../input';
+import { Input_Template_Switch, Input_Template_UserChatInput } from '../input';
 import { Output_Template_Finish } from '../output';
 import { DatasetSearchModeEnum } from '../../../dataset/constant';
 
 export const DatasetSearchModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.datasetSearchNode,
-  templateType: ModuleTemplateTypeEnum.dataset,
+  templateType: ModuleTemplateTypeEnum.functionCall,
   flowType: FlowNodeTypeEnum.datasetSearchNode,
   avatar: '/imgs/module/db.png',
   name: '知识库搜索',
   intro: '去知识库中搜索对应的答案。可作为 AI 对话引用参考。',
   showStatus: true,
   inputs: [
-    Input_Template_TFSwitch,
+    Input_Template_Switch,
     {
       key: ModuleInputKeyEnum.datasetSelectList,
       type: FlowNodeInputTypeEnum.selectDataset,
       label: '关联的知识库',
       value: [],
-      valueType: ModuleDataTypeEnum.selectDataset,
+      valueType: ModuleIOValueTypeEnum.selectDataset,
       list: [],
       required: true,
       showTargetInApp: false,
@@ -40,7 +40,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       type: FlowNodeInputTypeEnum.hidden,
       label: '最低相关性',
       value: 0.4,
-      valueType: ModuleDataTypeEnum.number,
+      valueType: ModuleIOValueTypeEnum.number,
       min: 0,
       max: 1,
       step: 0.01,
@@ -57,7 +57,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       label: '引用上限',
       description: '单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens',
       value: 1500,
-      valueType: ModuleDataTypeEnum.number,
+      valueType: ModuleIOValueTypeEnum.number,
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -65,7 +65,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.datasetSearchMode,
       type: FlowNodeInputTypeEnum.hidden,
       label: 'core.dataset.search.Mode',
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       showTargetInApp: false,
       showTargetInPlugin: false,
       value: DatasetSearchModeEnum.embedding
@@ -74,8 +74,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.datasetParamsModal,
       type: FlowNodeInputTypeEnum.selectDatasetParamsModal,
       label: '',
-      connected: false,
-      valueType: ModuleDataTypeEnum.any,
+      valueType: ModuleIOValueTypeEnum.any,
       showTargetInApp: false,
       showTargetInPlugin: false
     },
@@ -86,14 +85,14 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       key: ModuleOutputKeyEnum.datasetIsEmpty,
       label: '搜索结果为空',
       type: FlowNodeOutputTypeEnum.source,
-      valueType: ModuleDataTypeEnum.boolean,
+      valueType: ModuleIOValueTypeEnum.boolean,
       targets: []
     },
     {
       key: ModuleOutputKeyEnum.datasetUnEmpty,
       label: '搜索结果不为空',
       type: FlowNodeOutputTypeEnum.source,
-      valueType: ModuleDataTypeEnum.boolean,
+      valueType: ModuleIOValueTypeEnum.boolean,
       targets: []
     },
     {
@@ -102,7 +101,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       description:
         '始终返回数组，如果希望搜索结果为空时执行额外操作，需要用到上面的两个输入以及目标模块的触发器',
       type: FlowNodeOutputTypeEnum.source,
-      valueType: ModuleDataTypeEnum.datasetQuote,
+      valueType: ModuleIOValueTypeEnum.datasetQuote,
       targets: []
     },
     Output_Template_Finish

packages/global/core/module/template/system/http.ts

@@ -1,8 +1,16 @@
-import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
-import { FlowModuleTemplateType } from '../../type.d';
-import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
-import { Input_Template_TFSwitch } from '../input';
-import { Output_Template_Finish } from '../output';
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type';
+import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+import {
+  Input_Template_AddInputParam,
+  Input_Template_DynamicInput,
+  Input_Template_Switch
+} from '../input';
+import { Output_Template_AddOutput, Output_Template_Finish } from '../output';
 
 export const HttpModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.httpRequest,
@@ -13,19 +21,86 @@ export const HttpModule: FlowModuleTemplateType = {
   intro: '可以发出一个 HTTP POST 请求，实现更为复杂的操作（联网搜索、数据库查询等）',
   showStatus: true,
   inputs: [
-    Input_Template_TFSwitch,
+    Input_Template_Switch,
     {
-      key: ModuleInputKeyEnum.httpUrl,
-      value: '',
-      type: FlowNodeInputTypeEnum.input,
-      valueType: ModuleDataTypeEnum.string,
-      label: '请求地址',
-      description: '请求目标地址',
-      placeholder: 'https://api.ai.com/getInventory',
+      key: ModuleInputKeyEnum.httpMethod,
+      type: FlowNodeInputTypeEnum.select,
+      valueType: ModuleIOValueTypeEnum.string,
+      label: 'core.module.input.label.Http Request Method',
+      value: 'POST',
+      list: [
+        {
+          label: 'GET',
+          value: 'GET'
+        },
+        {
+          label: 'POST',
+          value: 'POST'
+        }
+      ],
       required: true,
       showTargetInApp: false,
       showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.httpReqUrl,
+      type: FlowNodeInputTypeEnum.input,
+      valueType: ModuleIOValueTypeEnum.string,
+      label: 'core.module.input.label.Http Request Url',
+      description: 'core.module.input.description.Http Request Url',
+      placeholder: 'https://api.ai.com/getInventory',
+      required: false,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.httpHeader,
+      type: FlowNodeInputTypeEnum.textarea,
+      valueType: ModuleIOValueTypeEnum.string,
+      label: 'core.module.input.label.Http Request Header',
+      description: 'core.module.input.description.Http Request Header',
+      placeholder: 'core.module.input.description.Http Request Header',
+      required: false,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    Input_Template_DynamicInput,
+    {
+      ...Input_Template_AddInputParam,
+      editField: {
+        key: true,
+        name: true,
+        description: true,
+        required: true,
+        dataType: true
+      },
+      defaultEditField: {
+        label: '',
+        key: '',
+        description: '',
+        inputType: FlowNodeInputTypeEnum.target,
+        valueType: ModuleIOValueTypeEnum.string,
+        required: true
+      }
     }
   ],
-  outputs: [Output_Template_Finish]
+  outputs: [
+    Output_Template_Finish,
+    {
+      ...Output_Template_AddOutput,
+      editField: {
+        key: true,
+        name: true,
+        description: true,
+        dataType: true
+      },
+      defaultEditField: {
+        label: '',
+        key: '',
+        description: '',
+        outputType: FlowNodeOutputTypeEnum.source,
+        valueType: ModuleIOValueTypeEnum.string
+      }
+    }
+  ]
 };

packages/global/core/module/template/system/runApp.ts

@@ -5,14 +5,14 @@ import {
 } from '../../node/constant';
 import { FlowModuleTemplateType } from '../../type.d';
 import {
-  ModuleDataTypeEnum,
+  ModuleIOValueTypeEnum,
   ModuleInputKeyEnum,
   ModuleOutputKeyEnum,
   ModuleTemplateTypeEnum
 } from '../../constants';
 import {
   Input_Template_History,
-  Input_Template_TFSwitch,
+  Input_Template_Switch,
   Input_Template_UserChatInput
 } from '../input';
 import { Output_Template_Finish } from '../output';
@@ -26,11 +26,11 @@ export const RunAppModule: FlowModuleTemplateType = {
   intro: '可以选择一个其他应用进行调用',
   showStatus: true,
   inputs: [
-    Input_Template_TFSwitch,
+    Input_Template_Switch,
     {
       key: ModuleInputKeyEnum.runAppSelectApp,
       type: FlowNodeInputTypeEnum.selectApp,
-      valueType: ModuleDataTypeEnum.selectApp,
+      valueType: ModuleIOValueTypeEnum.selectApp,
       label: '选择一个应用',
       description: '选择一个其他应用进行调用',
       required: true,
@@ -45,7 +45,7 @@ export const RunAppModule: FlowModuleTemplateType = {
       key: ModuleOutputKeyEnum.history,
       label: '新的上下文',
       description: '将该应用回复内容拼接到历史记录中，作为新的上下文返回',
-      valueType: ModuleDataTypeEnum.chatHistory,
+      valueType: ModuleIOValueTypeEnum.chatHistory,
       type: FlowNodeOutputTypeEnum.source,
       targets: []
     },
@@ -53,7 +53,7 @@ export const RunAppModule: FlowModuleTemplateType = {
       key: ModuleOutputKeyEnum.answerText,
       label: 'AI回复',
       description: '将在应用完全结束后触发',
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       type: FlowNodeOutputTypeEnum.source,
       targets: []
     },

packages/global/core/module/template/system/userGuide.ts

@@ -1,7 +1,7 @@
 import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
 import { FlowModuleTemplateType } from '../../type.d';
 import { userGuideTip } from '../tip';
-import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+import { ModuleIOValueTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
 
 export const UserGuideModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.userGuide,
@@ -14,7 +14,7 @@ export const UserGuideModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.welcomeText,
       type: FlowNodeInputTypeEnum.hidden,
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       label: '开场白',
       showTargetInApp: false,
       showTargetInPlugin: false
@@ -22,7 +22,7 @@ export const UserGuideModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.variables,
       type: FlowNodeInputTypeEnum.hidden,
-      valueType: ModuleDataTypeEnum.any,
+      valueType: ModuleIOValueTypeEnum.any,
       label: '对话框变量',
       value: [],
       showTargetInApp: false,
@@ -30,7 +30,7 @@ export const UserGuideModule: FlowModuleTemplateType = {
     },
     {
       key: ModuleInputKeyEnum.questionGuide,
-      valueType: ModuleDataTypeEnum.boolean,
+      valueType: ModuleIOValueTypeEnum.boolean,
       type: FlowNodeInputTypeEnum.switch,
       label: '问题引导',
       showTargetInApp: false,
@@ -39,7 +39,7 @@ export const UserGuideModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.tts,
       type: FlowNodeInputTypeEnum.hidden,
-      valueType: ModuleDataTypeEnum.any,
+      valueType: ModuleIOValueTypeEnum.any,
       label: '语音播报',
       showTargetInApp: false,
       showTargetInPlugin: false

packages/global/core/module/template/system/userInput.ts

@@ -5,7 +5,7 @@ import {
 } from '../../node/constant';
 import { FlowModuleTemplateType } from '../../type.d';
 import {
-  ModuleDataTypeEnum,
+  ModuleIOValueTypeEnum,
   ModuleInputKeyEnum,
   ModuleOutputKeyEnum,
   ModuleTemplateTypeEnum
@@ -22,7 +22,7 @@ export const UserInputModule: FlowModuleTemplateType = {
     {
       key: ModuleInputKeyEnum.userChatInput,
       type: FlowNodeInputTypeEnum.systemInput,
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       label: '用户问题',
       showTargetInApp: false,
       showTargetInPlugin: false
@@ -33,7 +33,7 @@ export const UserInputModule: FlowModuleTemplateType = {
       key: ModuleOutputKeyEnum.userChatInput,
       label: '用户问题',
       type: FlowNodeOutputTypeEnum.source,
-      valueType: ModuleDataTypeEnum.string,
+      valueType: ModuleIOValueTypeEnum.string,
       targets: []
     }
   ]

packages/global/core/module/type.d.ts

@@ -1,5 +1,5 @@
 import { FlowNodeTypeEnum } from './node/constant';
-import { ModuleDataTypeEnum, ModuleTemplateTypeEnum, VariableInputEnum } from './constants';
+import { ModuleIOValueTypeEnum, ModuleTemplateTypeEnum, VariableInputEnum } from './constants';
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
 
 export type FlowModuleTemplateType = {
@@ -72,4 +72,5 @@ export type ContextExtractAgentItemType = {
   desc: string;
   key: string;
   required: boolean;
+  enum?: string;
 };

packages/global/core/module/utils.ts

@@ -1,7 +1,8 @@
 import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from './node/constant';
-import { ModuleDataTypeEnum, ModuleInputKeyEnum } from './constants';
+import { ModuleIOValueTypeEnum, ModuleInputKeyEnum } from './constants';
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
 import { AppTTSConfigType, ModuleItemType, VariableItemType } from './type';
+import { Input_Template_Switch } from './template/input';
 
 export const getGuideModule = (modules: ModuleItemType[]) =>
   modules.find((item) => item.flowType === FlowNodeTypeEnum.userGuide);
@@ -29,42 +30,64 @@ export const splitGuideModule = (guideModules?: ModuleItemType) => {
   };
 };
 
-export function formatPluginToPreviewModule(
+export const getOrInitModuleInputValue = (input: FlowNodeInputItemType) => {
+  if (input.value !== undefined || !input.valueType) return input.value;
+
+  const map: Record<string, any> = {
+    [ModuleIOValueTypeEnum.boolean]: false,
+    [ModuleIOValueTypeEnum.number]: 0,
+    [ModuleIOValueTypeEnum.string]: ''
+  };
+
+  return map[input.valueType];
+};
+
+export const getModuleInputUiField = (input: FlowNodeInputItemType) => {
+  if (input.type === FlowNodeInputTypeEnum.input || input.type === FlowNodeInputTypeEnum.textarea) {
+    return {
+      placeholder: input.placeholder || input.description
+    };
+  }
+  return {};
+};
+
+export function plugin2ModuleIO(
   pluginId: string,
   modules: ModuleItemType[]
 ): {
   inputs: FlowNodeInputItemType[];
   outputs: FlowNodeOutputItemType[];
 } {
-  function getPluginTemplatePluginIdInput(pluginId: string): FlowNodeInputItemType {
-    return {
-      key: ModuleInputKeyEnum.pluginId,
-      type: FlowNodeInputTypeEnum.hidden,
-      label: 'pluginId',
-      value: pluginId,
-      valueType: ModuleDataTypeEnum.string,
-      connected: true,
-      showTargetInApp: false,
-      showTargetInPlugin: false
-    };
-  }
-
   const pluginInput = modules.find((module) => module.flowType === FlowNodeTypeEnum.pluginInput);
-  const customOutput = modules.find((module) => module.flowType === FlowNodeTypeEnum.pluginOutput);
+  const pluginOutput = modules.find((module) => module.flowType === FlowNodeTypeEnum.pluginOutput);
 
   return {
     inputs: pluginInput
       ? [
-          getPluginTemplatePluginIdInput(pluginId),
+          {
+            // plugin id
+            key: ModuleInputKeyEnum.pluginId,
+            type: FlowNodeInputTypeEnum.hidden,
+            label: 'pluginId',
+            value: pluginId,
+            valueType: ModuleIOValueTypeEnum.string,
+            connected: true,
+            showTargetInApp: false,
+            showTargetInPlugin: false
+          },
+          // switch
+          Input_Template_Switch,
           ...pluginInput.inputs.map((item) => ({
             ...item,
+            ...getModuleInputUiField(item),
+            value: getOrInitModuleInputValue(item),
             edit: false,
             connected: false
           }))
         ]
       : [],
-    outputs: customOutput
-      ? customOutput.outputs.map((item) => ({
+    outputs: pluginOutput
+      ? pluginOutput.outputs.map((item) => ({
           ...item,
           edit: false
         }))

packages/global/core/plugin/constants.ts

@@ -1,4 +1,3 @@
-import { ModuleTemplateTypeEnum } from '../module/constants';
 import { ModuleItemType } from '../module/type';
 
 export const defaultModules: ModuleItemType[] = [
@@ -28,13 +27,8 @@ export const defaultModules: ModuleItemType[] = [
   }
 ];
 
-export enum PluginTypeEnum {
+export enum PluginSourceEnum {
   personal = 'personal',
   community = 'community',
   commercial = 'commercial'
 }
-export const PluginType2TemplateTypeMap = {
-  [PluginTypeEnum.personal]: ModuleTemplateTypeEnum.personalPlugin,
-  [PluginTypeEnum.community]: ModuleTemplateTypeEnum.communityPlugin,
-  [PluginTypeEnum.commercial]: ModuleTemplateTypeEnum.commercialPlugin
-};

packages/global/core/plugin/type.d.ts

@@ -1,6 +1,6 @@
 import { ModuleTemplateTypeEnum } from 'core/module/constants';
-import type { ModuleItemType } from '../module/type.d';
-import { PluginTypeEnum } from './constants';
+import type { FlowModuleTemplateType, ModuleItemType } from '../module/type.d';
+import { PluginSourceEnum } from './constants';
 
 export type PluginItemSchema = {
   _id: string;
@@ -16,11 +16,13 @@ export type PluginItemSchema = {
 
 /* plugin template */
 export type PluginTemplateType = {
+  author?: string;
   id: string;
-  type: `${PluginTypeEnum}`;
+  source: `${PluginSourceEnum}`;
+  templateType: FlowModuleTemplateType['templateType'];
   name: string;
   avatar: string;
   intro: string;
+  showStatus?: boolean;
   modules: ModuleItemType[];
-  templateType?: `${ModuleTemplateTypeEnum}`;
 };

packages/global/package.json

@@ -2,17 +2,14 @@
   "name": "@fastgpt/global",
   "version": "1.0.0",
   "dependencies": {
-    "axios": "^1.5.1",
-    "cheerio": "1.0.0-rc.12",
     "dayjs": "^1.11.7",
+    "openai": "4.23.0",
     "encoding": "^0.1.13",
     "js-tiktoken": "^1.0.7",
-    "node-html-markdown": "^1.3.0",
-    "openai": "^4.16.1",
+    "axios": "^1.5.1",
     "timezones-list": "^3.0.2"
   },
   "devDependencies": {
-    "@types/node": "^20.8.5",
-    "@types/turndown": "^5.0.4"
+    "@types/node": "^20.8.5"
   }
 }

packages/global/support/user/constant.ts

@@ -1,4 +1,22 @@
+export enum UserStatusEnum {
+  active = 'active',
+  forbidden = 'forbidden'
+}
+export const userStatusMap = {
+  [UserStatusEnum.active]: {
+    label: 'support.user.status.active'
+  },
+  [UserStatusEnum.forbidden]: {
+    label: 'support.user.status.forbidden'
+  }
+};
+
 export enum OAuthEnum {
   github = 'github',
   google = 'google'
 }
+
+export enum UserAuthTypeEnum {
+  register = 'register',
+  findPassword = 'findPassword'
+}

packages/global/support/user/type.d.ts

@@ -1,4 +1,4 @@
-import { InformTypeEnum } from './constant';
+import { InformTypeEnum, UserStatusEnum } from './constant';
 import { TeamItemType } from './team/type';
 
 export type UserModelSchema = {
@@ -12,6 +12,7 @@ export type UserModelSchema = {
   openaiKey: string;
   createTime: number;
   timezone: string;
+  status: `${UserStatusEnum}`;
   openaiAccount?: {
     key: string;
     baseUrl: string;

packages/plugins/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "@fastgpt/plugins",
+  "version": "1.0.0",
+  "dependencies": {},
+  "devDependencies": {
+    "@types/node": "^20.8.5",
+    "@fastgpt/global": "workspace:*",
+    "@fastgpt/service": "workspace:*"
+  }
+}

packages/plugins/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "es2015",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "baseUrl": "."
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"],
+  "exclude": ["node_modules"]
+}

packages/service/common/api/plusRequest.ts

@@ -70,8 +70,9 @@ instance.interceptors.request.use(requestStart, (err) => Promise.reject(err));
 instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err));
 
 export function request(url: string, data: any, config: ConfigType, method: Method): any {
-  if (!global.systemEnv?.pluginBaseUrl) {
-    return Promise.reject('该功能为商业版特有...');
+  if (!global.systemEnv || !global.systemEnv?.pluginBaseUrl) {
+    console.log('未部署商业版接口', url);
+    return Promise.reject('The The request was denied...');
   }
 
   /* 去空 */

packages/service/common/file/image/schema.ts

@@ -8,6 +8,10 @@ const ImageSchema = new Schema({
     ref: TeamCollectionName,
     required: true
   },
+  createTime: {
+    type: Date,
+    default: () => new Date()
+  },
   binary: {
     type: Buffer
   },
@@ -25,7 +29,10 @@ try {
   console.log(error);
 }
 
-export const MongoImage: Model<{ teamId: string; binary: Buffer; metadata?: Record<string, any> }> =
-  models['image'] || model('image', ImageSchema);
+export const MongoImage: Model<{
+  teamId: string;
+  binary: Buffer;
+  metadata?: { fileId?: string };
+}> = models['image'] || model('image', ImageSchema);
 
 MongoImage.syncIndexes();

packages/service/common/mongo/controller.ts

@@ -1,26 +0,0 @@
-/* add logger */
-export const addLog = {
-  info: (msg: string, obj?: Record<string, any>) => {
-    global.logger?.info(msg, { meta: obj });
-  },
-  error: (msg: string, error?: any) => {
-    global.logger?.error(msg, {
-      meta: {
-        stack: error?.stack,
-        ...(error?.config && {
-          config: {
-            headers: error.config.headers,
-            url: error.config.url,
-            data: error.config.data
-          }
-        }),
-        ...(error?.response && {
-          response: {
-            status: error.response.status,
-            statusText: error.response.statusText
-          }
-        })
-      }
-    });
-  }
-};

packages/service/common/mongo/init.ts

@@ -1,6 +1,4 @@
 import mongoose from './index';
-import { createLogger, format, transports } from 'winston';
-import 'winston-mongodb';
 
 /**
  * connect MongoDB and init data
@@ -36,7 +34,6 @@ export async function connectMongo({
     });
 
     console.log('mongo connected');
-    initLogger();
 
     afterHook && (await afterHook());
   } catch (error) {
@@ -45,35 +42,3 @@ export async function connectMongo({
     global.mongodb = undefined;
   }
 }
-
-function initLogger() {
-  global.logger = createLogger({
-    transports: [
-      new transports.MongoDB({
-        db: process.env.MONGODB_URI as string,
-        collection: 'server_logs',
-        options: {
-          useUnifiedTopology: true
-        },
-        cappedSize: 500000000,
-        tryReconnect: true,
-        metaKey: 'meta',
-        format: format.combine(format.timestamp(), format.json())
-      }),
-      new transports.Console({
-        format: format.combine(
-          format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
-          format.printf((info) => {
-            if (info.level === 'error') {
-              console.log(info.meta);
-              return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`;
-            }
-            return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${
-              info.meta ? `: ${JSON.stringify(info.meta)}` : ''
-            }`;
-          })
-        )
-      })
-    ]
-  });
-}

packages/service/common/mongo/type.d.ts

@@ -3,5 +3,4 @@ import type { Logger } from 'winston';
 
 declare global {
   var mongodb: Mongoose | undefined;
-  var logger: Logger;
 }

packages/service/common/pg/index.ts

@@ -171,9 +171,10 @@ export async function initPg() {
           tmb_id VARCHAR(50) NOT NULL,
           dataset_id VARCHAR(50) NOT NULL,
           collection_id VARCHAR(50) NOT NULL,
-          data_id VARCHAR(50) NOT NULL
+          data_id VARCHAR(50) NOT NULL,
+          createTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP
       );
-      CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 24, ef_construction = 64);
+      CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);
     `);
 
     console.log('init pg successful');

packages/service/common/response/index.ts

@@ -1,7 +1,7 @@
 import type { NextApiResponse } from 'next';
 import { sseResponseEventEnum } from './constant';
 import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
-import { addLog } from '../mongo/controller';
+import { addLog } from '../system/log';
 import { clearCookie } from '../../support/permission/controller';
 
 export interface ResponseType<T = any> {

packages/service/common/string/cheerio.ts

@@ -0,0 +1,95 @@
+import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api';
+import * as cheerio from 'cheerio';
+import axios from 'axios';
+import { htmlToMarkdown } from './markdown';
+
+export const cheerioToHtml = ({
+  fetchUrl,
+  $,
+  selector
+}: {
+  fetchUrl: string;
+  $: cheerio.CheerioAPI;
+  selector?: string;
+}) => {
+  // get origin url
+  const originUrl = new URL(fetchUrl).origin;
+
+  const selectDom = $(selector || 'body');
+
+  // remove i element
+  selectDom.find('i,script').remove();
+
+  // remove empty a element
+  selectDom
+    .find('a')
+    .filter((i, el) => {
+      return $(el).text().trim() === '' && $(el).children().length === 0;
+    })
+    .remove();
+
+  // if link,img startWith /, add origin url
+  selectDom.find('a').each((i, el) => {
+    const href = $(el).attr('href');
+    if (href && href.startsWith('/')) {
+      $(el).attr('href', originUrl + href);
+    }
+  });
+  selectDom.find('img').each((i, el) => {
+    const src = $(el).attr('src');
+    if (src && src.startsWith('/')) {
+      $(el).attr('src', originUrl + src);
+    }
+  });
+
+  const html = selectDom
+    .map((item, dom) => {
+      return $(dom).html();
+    })
+    .get()
+    .join('\n');
+
+  return html;
+};
+export const urlsFetch = async ({
+  urlList,
+  selector
+}: UrlFetchParams): Promise<UrlFetchResponse> => {
+  urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
+
+  const response = (
+    await Promise.all(
+      urlList.map(async (url) => {
+        try {
+          const fetchRes = await axios.get(url, {
+            timeout: 30000
+          });
+
+          const $ = cheerio.load(fetchRes.data);
+
+          const md = await htmlToMarkdown(
+            cheerioToHtml({
+              fetchUrl: url,
+              $,
+              selector
+            })
+          );
+
+          return {
+            url,
+            content: md
+          };
+        } catch (error) {
+          console.log(error, 'fetch error');
+
+          return {
+            url,
+            content: ''
+          };
+        }
+      })
+    )
+  ).filter((item) => item.content);
+
+  return response;
+};

packages/service/common/string/markdown.ts

@@ -0,0 +1,26 @@
+import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
+import { Worker } from 'worker_threads';
+import { getWorkerPath } from './utils';
+
+/* html string to markdown */
+export const htmlToMarkdown = (html?: string | null) =>
+  new Promise<string>((resolve, reject) => {
+    if (!html) return resolve('');
+
+    const start = Date.now();
+
+    // worker
+    const worker = new Worker(getWorkerPath('html2md'));
+
+    worker.on('message', (md: string) => {
+      worker.terminate();
+
+      resolve(simpleMarkdownText(md));
+    });
+    worker.on('error', (err) => {
+      worker.terminate();
+      reject(err);
+    });
+
+    worker.postMessage(html);
+  });

packages/service/common/string/utils.ts

@@ -0,0 +1,9 @@
+export const getWorkerPath = (name: string) => {
+  // @ts-ignore
+  const isSubModule = !!global?.systemConfig;
+
+  const isProd = process.env.NODE_ENV === 'production';
+  return isProd
+    ? `/app/worker/${name}.js`
+    : `../../${isSubModule ? 'FastGPT/' : ''}/worker/${name}.js`;
+};

packages/service/common/system/config/controller.ts

@@ -0,0 +1,15 @@
+import { SystemConfigsTypeEnum } from '@fastgpt/global/common/system/config/constants';
+import { MongoSystemConfigs } from './schema';
+import { FeConfigsType } from '@fastgpt/global/common/system/types';
+
+export const getFastGPTFeConfig = async () => {
+  const res = await MongoSystemConfigs.findOne({
+    type: SystemConfigsTypeEnum.fastgpt
+  }).sort({
+    createTime: -1
+  });
+
+  const config: FeConfigsType = res?.value?.FeConfig || {};
+
+  return config;
+};

packages/service/common/system/config/schema.ts

@@ -19,14 +19,15 @@ const systemConfigSchema = new Schema({
     type: Date,
     default: () => new Date()
   }
-})
+});
 
 try {
   systemConfigSchema.index({ createTime: -1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 });
+  systemConfigSchema.index({ type: 1 });
 } catch (error) {
   console.log(error);
 }
 
-export const MongoSystemConfigs: Model<SystemConfigsType>=
+export const MongoSystemConfigs: Model<SystemConfigsType> =
   models[collectionName] || model(collectionName, systemConfigSchema);
-  MongoSystemConfigs.syncIndexes();
+MongoSystemConfigs.syncIndexes();

packages/service/common/system/log.ts

@@ -0,0 +1,66 @@
+import dayjs from 'dayjs';
+
+/* add logger */
+export const addLog = {
+  log(level: 'info' | 'warn' | 'error', msg: string, obj: Record<string, any> = {}) {
+    console.log(
+      `[${level.toLocaleUpperCase()}] ${dayjs().format(
+        'YYYY-MM-DD HH:mm:ss'
+      )} ${msg}: ${JSON.stringify(obj)}`
+    );
+
+    const lokiUrl = process.env.LOKI_LOG_URL as string;
+    if (!lokiUrl) return;
+
+    try {
+      fetch(lokiUrl, {
+        method: 'POST',
+        headers: {
+          'Content-type': 'application/json'
+        },
+        body: JSON.stringify({
+          streams: [
+            {
+              stream: {
+                level
+              },
+              values: [
+                [
+                  `${Date.now() * 1000000}`,
+                  JSON.stringify({
+                    message: msg,
+                    ...obj
+                  })
+                ]
+              ]
+            }
+          ]
+        })
+      });
+    } catch (error) {}
+  },
+  info(msg: string, obj?: Record<string, any>) {
+    this.log('info', msg, obj);
+  },
+  warn(msg: string, obj?: Record<string, any>) {
+    this.log('warn', msg, obj);
+  },
+  error(msg: string, error?: any) {
+    this.log('error', msg, {
+      stack: error?.stack,
+      ...(error?.config && {
+        config: {
+          headers: error.config.headers,
+          url: error.config.url,
+          data: error.config.data
+        }
+      }),
+      ...(error?.response && {
+        response: {
+          status: error.response.status,
+          statusText: error.response.statusText
+        }
+      })
+    });
+  }
+};

packages/service/core/ai/functions/createQuestionGuide.ts

@@ -10,7 +10,7 @@ export async function createQuestionGuide({
   messages: ChatMessageItemType[];
   model: string;
 }) {
-  const ai = getAIApi(undefined, 48000);
+  const ai = getAIApi(undefined, 480000);
   const data = await ai.chat.completions.create({
     model: model,
     temperature: 0,

packages/service/core/chat/chatItemSchema.ts

@@ -54,9 +54,16 @@ const ChatItemSchema = new Schema({
     type: String,
     default: ''
   },
-  userFeedback: {
+  userGoodFeedback: {
     type: String
   },
+  userFeedback: String,
+  userBadFeedback: {
+    type: String
+  },
+  customFeedbacks: {
+    type: [String]
+  },
   adminFeedback: {
     type: {
       datasetId: String,
@@ -77,7 +84,10 @@ try {
   ChatItemSchema.index({ userId: 1 });
   ChatItemSchema.index({ appId: 1 });
   ChatItemSchema.index({ chatId: 1 });
-  ChatItemSchema.index({ userFeedback: 1 });
+  ChatItemSchema.index({ userGoodFeedback: 1 });
+  ChatItemSchema.index({ userBadFeedback: 1 });
+  ChatItemSchema.index({ customFeedbacks: 1 });
+  ChatItemSchema.index({ adminFeedback: 1 });
 } catch (error) {
   console.log(error);
 }

packages/service/core/chat/controller.ts

@@ -1,5 +1,6 @@
 import type { ChatItemType } from '@fastgpt/global/core/chat/type';
 import { MongoChatItem } from './chatItemSchema';
+import { addLog } from '../../common/system/log';
 
 export async function getChatItems({
   chatId,
@@ -14,9 +15,35 @@ export async function getChatItems({
     return { history: [] };
   }
 
-  const history = await MongoChatItem.find({ chatId }, field).sort({ _id: -1 }).limit(limit);
+  const history = await MongoChatItem.find({ chatId }, field).sort({ _id: -1 }).limit(limit).lean();
 
   history.reverse();
 
   return { history };
 }
+
+export const addCustomFeedbacks = async ({
+  chatId,
+  chatItemId,
+  feedbacks
+}: {
+  chatId?: string;
+  chatItemId?: string;
+  feedbacks: string[];
+}) => {
+  if (!chatId || !chatItemId) return;
+
+  try {
+    await MongoChatItem.findOneAndUpdate(
+      {
+        chatId,
+        dataId: chatItemId
+      },
+      {
+        $push: { customFeedbacks: { $each: feedbacks } }
+      }
+    );
+  } catch (error) {
+    addLog.error('addCustomFeedbacks error', error);
+  }
+};

packages/service/core/dataset/collection/controller.ts

@@ -6,6 +6,8 @@ import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset
 import { MongoDatasetCollection } from './schema';
 
 export async function createOneCollection({
+  teamId,
+  tmbId,
   name,
   parentId,
   datasetId,
@@ -14,8 +16,8 @@ export async function createOneCollection({
   chunkSize = 0,
   fileId,
   rawLink,
-  teamId,
-  tmbId,
+  qaPrompt,
+  hashRawText,
   metadata = {}
 }: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
   const { _id } = await MongoDatasetCollection.create({
@@ -29,6 +31,8 @@ export async function createOneCollection({
     chunkSize,
     fileId,
     rawLink,
+    qaPrompt,
+    hashRawText,
     metadata
   });
 
@@ -71,3 +75,19 @@ export function createDefaultCollection({
     updateTime: new Date('2099')
   });
 }
+
+// check same collection
+export const getSameRawTextCollection = async ({
+  datasetId,
+  hashRawText
+}: {
+  datasetId: string;
+  hashRawText?: string;
+}) => {
+  const collection = await MongoDatasetCollection.findOne({
+    datasetId,
+    hashRawText
+  });
+
+  return collection;
+};