perf: input guide (#1558)

* perf: input guide code

* perf: input guide ui

* Chat input guide api

* Update app chat config store

* perf: app chat config field

* perf: app context

* perf: params

* fix: ts

* perf: filter private config

* perf: filter private config

* perf: import workflow

* perf: limit max tip amount

.npmrc

@@ -0,0 +1 @@
+public-hoist-pattern[]=*tiktoken*

.vscode/i18n-ally-custom-framework.yml

@@ -23,6 +23,9 @@ usageMatchRegex:
   - "[^\\w\\d]datasetT\\(['\"`]({key})['\"`]"
   - "[^\\w\\d]fileT\\(['\"`]({key})['\"`]"
   - "[^\\w\\d]publishT\\(['\"`]({key})['\"`]"
+  - "[^\\w\\d]workflowT\\(['\"`]({key})['\"`]"
+  - "[^\\w\\d]userT\\(['\"`]({key})['\"`]"
+  - "[^\\w\\d]chatT\\(['\"`]({key})['\"`]"
 
 # A RegEx to set a custom scope range. This scope will be used as a prefix when detecting keys
 # and works like how the i18next framework identifies the namespace scope from the

.vscode/nextapi.code-snippets

@@ -10,14 +10,19 @@
 		"scope": "javascript,typescript",
 		"prefix": "nextapi",
 		"body": [
-			"import type { NextApiRequest, NextApiResponse } from 'next';",
+			"import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';",
-			"import { NextAPI } from '@/service/middle/entry';",
+			"import { NextAPI } from '@/service/middleware/entry';",
 			"",
-			"type Props = {};",
+			"export type ${TM_FILENAME_BASE}Query = {};",
 			"",
-			"type Response = {};",
+			"export type ${TM_FILENAME_BASE}Body = {};",
 			"",
-			"async function handler(req: NextApiRequest, res: NextApiResponse<any>): Promise<Response> {",
+			"export type ${TM_FILENAME_BASE}Response = {};",
+			"",
+			"async function handler(",
+			"  req: ApiRequestProps<${TM_FILENAME_BASE}Body, ${TM_FILENAME_BASE}Query>,",
+			"  res: ApiResponseType<any>",
+			"): Promise<${TM_FILENAME_BASE}Response> {",
 			"    $1",
 			"    return {}",
 			"}",
@@ -25,5 +30,23 @@
 			"export default NextAPI(handler);"
 		],
 		"description": "FastGPT Next API template"
+	},
+	"use context template": {
+		"scope": "typescriptreact",
+		"prefix": "context",
+		"body": [
+			"import { ReactNode } from 'react';",
+			"import { createContext } from 'use-context-selector';",
+			"",
+			"type ContextType = {$1};",
+			"",
+			"export const Context = createContext<ContextType>({});",
+			"",
+			"export const ContextProvider = ({ children }: { children: ReactNode }) => {",
+			"  const contextValue: ContextType = {};",
+			"  return <Context.Provider value={contextValue}>{children}</Context.Provider>;",
+			"};",
+		],
+		"description": "FastGPT usecontext template"
 	}
 }

.vscode/settings.json

@@ -11,5 +11,6 @@
     "i18n-ally.sortKeys": true,
     "i18n-ally.keepFulfilled": false,
     "i18n-ally.sourceLanguage": "zh", // 根据此语言文件翻译其他语言文件的变量和内容
-    "i18n-ally.displayLanguage": "zh" // 显示语言
+    "i18n-ally.displayLanguage": "zh", // 显示语言
+    "i18n-ally.extract.targetPickingStrategy": "most-similar-by-key"
 }

Dockerfile

@@ -11,7 +11,7 @@ RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
 RUN [ -z "$proxy" ] || pnpm config set registry https://registry.npmmirror.com
 
 # copy packages and one project
-COPY pnpm-lock.yaml pnpm-workspace.yaml ./
+COPY pnpm-lock.yaml pnpm-workspace.yaml .npmrc ./
 COPY ./packages ./packages
 COPY ./projects/$name/package.json ./projects/$name/package.json
 
@@ -27,7 +27,7 @@ ARG name
 ARG proxy
 
 # copy common node_modules and one project node_modules
-COPY package.json pnpm-workspace.yaml ./
+COPY package.json pnpm-workspace.yaml .npmrc ./
 COPY --from=mainDeps /app/node_modules ./node_modules
 COPY --from=mainDeps /app/packages ./packages
 COPY ./projects/$name ./projects/$name
@@ -36,6 +36,8 @@ COPY --from=mainDeps /app/projects/$name/node_modules ./projects/$name/node_modu
 RUN [ -z "$proxy" ] || sed -i 's/dl-cdn.alpinelinux.org/mirrors.ustc.edu.cn/g' /etc/apk/repositories
 
 RUN apk add --no-cache libc6-compat && npm install -g pnpm@8.6.0
+
+ENV NODE_OPTIONS="--max-old-space-size=4096"
 RUN pnpm --filter=$name build
 
 # --------- runner -----------
@@ -62,6 +64,11 @@ COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/static /app/
 COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/server/chunks /app/projects/$name/.next/server/chunks
 # copy worker
 COPY --from=builder --chown=nextjs:nodejs /app/projects/$name/.next/server/worker /app/projects/$name/.next/server/worker
+
+# copy tiktoken but not copy ./node_modules/tiktoken/encoders
+COPY --from=mainDeps /app/node_modules/tiktoken ./node_modules/tiktoken
+RUN rm -rf ./node_modules/tiktoken/encoders
+
 # copy package.json to version file
 COPY --from=builder /app/projects/$name/package.json ./package.json 
 # copy config
@@ -79,4 +86,4 @@ USER nextjs
 
 ENV serverPath=./projects/$name/server.js
 
-ENTRYPOINT ["sh","-c","node ${serverPath}"]
+ENTRYPOINT ["sh","-c","node --max-old-space-size=4096 ${serverPath}"]

docSite/content/docs/course/chat_input_guide.md

@@ -0,0 +1,50 @@
+---
+title: "对话问题引导"
+description: "FastGPT 对话问题引导"
+icon: "code"
+draft: false
+toc: true
+weight: 350
+---
+
+![](/imgs/questionGuide.png)
+
+## 什么是自定义问题引导
+
+你可以为你的应用提前预设一些问题，用户在输入时，会根据输入的内容，动态搜索这些问题作为提示，从而引导用户更快的进行提问。
+
+你可以直接在 FastGPT 中配置词库，或者提供自定义词库接口。
+
+## 自定义词库接口
+
+需要保证这个接口可以被用户浏览器访问。
+
+**请求：**
+
+```bash
+curl --location --request GET 'http://localhost:3000/api/core/chat/inputGuide/query?appId=663c75302caf8315b1c00194&searchKey=你'
+```
+
+**响应**
+
+```json
+{
+  "code": 200,
+  "statusText": "",
+  "message": "",
+  "data": [
+    "是你",
+    "你是谁呀",
+    "你好好呀",
+    "你好呀",
+    "你是谁！",
+    "你好"
+  ]
+}
+```
+
+
+**参数说明：**
+
+- appId - 应用ID
+- searchKey - 搜索关键字

docSite/content/docs/course/externalFile.md

@@ -0,0 +1,26 @@
+---
+title: '外部文件知识库'
+description: 'FastGPT 外部文件知识库功能介绍和使用方式'
+icon: 'language'
+draft: false
+toc: true
+weight: 107
+---
+
+外部文件库是 FastGPT 商业版特有功能。它允许接入你现在的文件系统，无需将文件再导入一份到 FastGPT 中。
+
+并且，阅读权限可以通过你的文件系统进行控制。
+
+|                       |                       | |
+| --------------------- | --------------------- | --------------------- |
+| ![](/imgs/external_file0.png) | ![](/imgs/external_file1.png) | ![](/imgs/external_file2.png) | 
+
+
+## 导入参数说明
+
+- 外部预览地址：用于跳转你的文件阅读地址，会携带“文件阅读ID”进行访问。
+- 文件访问URL：文件可访问的地址。
+- 文件阅读ID：通常情况下，文件访问URL是临时的。如果希望永久可以访问，你需要使用该文件阅读ID，并配合上“外部预览地址”，跳转至新的阅读地址进行原文件访问。
+- 文件名：默认会自动解析文件访问URL上的文件名。如果你手动填写，将会以手动填写的值为准。
+
+[点击查看API导入文档](/docs/development/openapi/dataset/#创建一个外部文件库集合商业版)

docSite/content/docs/development/faq.md

@@ -118,4 +118,5 @@ OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并
 ### bad_response_status_code bad response status code 503
 
 1. 模型服务不可用
-2. ....
+2. 模型接口参数异常（温度、max token等可能不适配）
+3. ....

docSite/content/docs/development/intro.md

@@ -133,3 +133,57 @@ FastGPT 在`pnpm i`后会执行`postinstall`脚本，用于自动生成`ChakraUI
 遇到困难了吗？有任何问题吗? 加入微信群与开发者和用户保持沟通。
 
 <img width="400px" src="https://oss.laf.run/htr4n1-images/fastgpt-qr-code.jpg" class="medium-zoom-image" />
+
+## 代码结构说明
+
+### nextjs
+
+FastGPT 使用了 nextjs 的 page route 作为框架。为了区分好前后端代码，在目录分配上会分成 global, service, web 3个自目录，分别对应着 `前后端共用`、`后端专用`、`前端专用`的代码。
+
+### monorepo
+FastGPT 采用 pnpm workspace 方式构建 monorepo 项目，主要分为两个部分：
+
+- projects/app - FastGPT 主项目  
+- packages/ - 子模块  
+  - global - 共用代码，通常是放一些前后端都能执行的函数、类型声明、常量。  
+  - service - 服务端代码  
+  - web - 前端代码  
+  - plugin - 工作流自定义插件的代码  
+
+### 领域驱动模式（DDD）
+
+FastGPT 在代码模块划分时，按DDD的思想进行划分，主要分为以下几个领域：
+
+core - 核心功能（知识库，工作流，应用，对话）
+support - 支撑功能（用户体系，计费，鉴权等）
+common - 基础功能（日志管理，文件读写等）
+
+{{% details title="代码结构说明" closed="true" %}}
+```
+.
+├── .github                      // github 相关配置
+├── .husky                       // 格式化配置
+├── docSite                      // 文档
+├── files                        // 一些外部文件，例如 docker-compose, helm
+├── packages                     // 子包
+│   ├── global                   // 前后端通用子包
+│   ├── plugins                  // 工作流插件（需要自定义包时候使用到）
+│   ├── service                  // 后端子包
+│   └── web                      // 前端子包
+├── projects
+│   └── app                      // FastGPT 主项目
+├── python                       // 存放一些模型代码，和 FastGPT 本身无关
+└── scripts                      // 一些自动化脚本
+    ├── icon                     // icon预览脚本，可以在顶层 pnpm initIcon(把svg写入到代码中), pnpm previewIcon（预览icon）
+    └── postinstall.sh           // chakraUI自定义theme初始化 ts 类型
+├── package.json                 // 顶层monorepo
+├── pnpm-lock.yaml
+├── pnpm-workspace.yaml          // monorepo 声明
+├── Dockerfile
+├── LICENSE
+├── README.md
+├── README_en.md
+├── README_ja.md
+├── dev.md
+```
+{{% /details %}}

docSite/content/docs/development/migration/ docker_mongo.md

@@ -1,10 +1,9 @@
 ---
-weight: 762
 title: "Docker Mongo迁移(dump模式)"
 description: "FastGPT Docker Mongo迁移"
 icon: database
 draft: false
-images: []
+weight: 762
 ---
 
 ## 作者

docSite/content/docs/development/openapi/dataset.md

@@ -295,6 +295,24 @@ curl --location --request DELETE 'http://localhost:3000/api/core/dataset/delete?
 
 ## 集合
 
+### 通用创建参数说明
+
+**入参**
+
+| 参数 | 说明 | 必填 |
+| --- | --- | --- |
+| datasetId | 知识库ID | ✅ |
+| parentId： | 父级ID，不填则默认为根目录 |  |
+| trainingType | 训练模式。chunk: 按文本长度进行分割;qa: QA拆分;auto: 增强训练 | ✅ |
+| chunkSize | 预估块大小 |  |
+| chunkSplitter | 自定义最高优先分割符号 |  |
+| qaPrompt | qa拆分提示词 |  |
+
+**出参**
+
+- collectionId - 新建的集合ID  
+- insertLen：插入的块数量
+
 ### 创建一个空的集合
 
 {{< tabs tabTotal="3" >}}
@@ -500,7 +518,7 @@ data 为集合的 ID。
 {{< /tab >}}
 {{< /tabs >}}
 
-### 创建一个文件集合(商业版)
+### 创建一个文件集合
 
 传入一个文件，创建一个集合，会读取文件内容进行分割。目前支持：pdf, docx, md, txt, html, csv。
 
@@ -509,7 +527,7 @@ data 为集合的 ID。
 {{< markdownify >}}
 
 ```bash
-curl --location --request POST 'http://localhost:3000/api/proApi/core/dataset/collection/create/file' \
+curl --location --request POST 'http://localhost:3000/api/core/dataset/collection/create/localFile' \
 --header 'Authorization: Bearer {{authorization}}' \
 --form 'file=@"C:\\Users\\user\\Desktop\\fastgpt测试文件\\index.html"' \
 --form 'data="{\"datasetId\":\"6593e137231a2be9c5603ba7\",\"parentId\":null,\"trainingType\":\"chunk\",\"chunkSize\":512,\"chunkSplitter\":\"\",\"qaPrompt\":\"\",\"metadata\":{}}"'
@@ -565,6 +583,68 @@ data 为集合的 ID。
 {{< /tab >}}
 {{< /tabs >}}
 
+### 创建一个外部文件库集合（商业版）
+
+{{< tabs tabTotal="3" >}}
+{{< tab tabName="请求示例" >}}
+{{< markdownify >}}
+
+```bash
+curl --location --request POST 'http://localhost:3000/api/proApi/core/dataset/collection/create/externalFileUrl' \
+--header 'Authorization: Bearer {{authorization}}' \
+--header 'User-Agent: Apifox/1.0.0 (https://apifox.com)' \
+--header 'Content-Type: application/json' \
+--data-raw '{
+    "externalFileUrl":"https://image.xxxxx.com/fastgpt-dev/%E6%91%82.pdf",
+    "externalFileId":"1111",
+    "filename":"自定义文件名",
+    "datasetId":"6642d105a5e9d2b00255b27b",
+    "parentId": null,
+
+    "trainingType": "chunk",
+    "chunkSize":512,
+    "chunkSplitter":"",
+    "qaPrompt":""
+}'
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="参数说明" >}}
+{{< markdownify >}}
+
+| 参数 | 说明 | 必填 |
+| --- | --- | --- |
+| externalFileUrl | 文件访问链接（可以是临时链接） | ✅ |
+| externalFileId | 外部文件ID |  |
+| filename | 自定义文件名 |  |
+
+
+{{< /markdownify >}}
+{{< /tab >}}
+
+{{< tab tabName="响应示例" >}}
+{{< markdownify >}}
+
+data 为集合的 ID。
+
+```json
+{
+  "code": 200,
+  "statusText": "",
+  "message": "",
+  "data": {
+    "collectionId": "6646fcedfabd823cdc6de746",
+    "insertLen": 3
+  }
+}
+```
+
+{{< /markdownify >}}
+{{< /tab >}}
+{{< /tabs >}}
+
 ### 获取集合列表
 
 {{< tabs tabTotal="3" >}}

docSite/content/docs/development/openapi/share.md

@@ -247,7 +247,7 @@ curl --location --request POST '{{host}}/shareAuth/finish' \
 
 ```ts
 type ResponseType = {
-    moduleType: `${FlowNodeTypeEnum}`; // 模块类型
+    moduleType: FlowNodeTypeEnum; // 模块类型
     moduleName: string; // 模块名
     moduleLogo?: string; // logo
     runningTime?: number; // 运行时间

docSite/content/docs/development/upgrading/48.md

@@ -1,5 +1,5 @@
 ---
-title: 'V4.8(开发中)'
+title: 'V4.8'
 description: 'FastGPT V4.8 更新说明'
 icon: 'upgrade'
 draft: false

docSite/content/docs/development/upgrading/481.md

@@ -0,0 +1,45 @@
+---
+title: 'V4.8.1(进行中)'
+description: 'FastGPT V4.8.1 更新说明'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 823
+---
+
+## 初始化脚本
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成FastGPT的域名。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv481' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+由于之前集合名不规范，该初始化会重置表名。请在初始化前，确保 dataset.trainings 表没有数据。
+最好更新该版本时，暂停所有进行中业务，再进行初始化，避免数据冲突。
+
+## 执行脏数据清理
+
+从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成FastGPT的域名。
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/clearInvalidData' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化完后，可以执行这个命令。之前定时清理的定时器有些问题，部分数据没被清理，可以手动执行清理。
+
+## V4.8.1 更新说明
+
+1. 新增 - 知识库重新选择向量模型重建
+2. 新增 - 对话框支持问题模糊检索提示，可自定义预设问题词库。
+3. 新增 - 工作流节点版本变更提示，并可以同步最新版本配置，避免存在隐藏脏数据。
+4. 新增 - 开放文件导入知识库接口到开源版， [点击插件文档](/docs/development/openapi/dataset/#创建一个文件集合)
+5. 新增 - 外部文件源知识库, [点击查看文档](/docs/course/externalfile/)
+6. 优化 - 插件输入的 debug 模式，支持全量参数输入渲染。
+7. 修复 - 插件输入默认值被清空问题。
+8. 修复 - 工作流删除节点的动态输入和输出时候，没有正确的删除连接线，导致可能出现逻辑异常。
+9. 修复 - 定时器清理脏数据任务

packages/global/common/string/textSplitter.ts

@@ -9,6 +9,9 @@ type SplitProps = {
   overlapRatio?: number;
   customReg?: string[];
 };
+export type TextSplitProps = Omit<SplitProps, 'text' | 'chunkLen'> & {
+  chunkLen?: number;
+};
 
 type SplitResponse = {
   chunks: string[];
@@ -49,6 +52,7 @@ const strIsMdTable = (str: string) => {
       return false;
     }
   }
+
   return true;
 };
 const markdownTableSplit = (props: SplitProps): SplitResponse => {
@@ -77,6 +81,10 @@ ${mdSplitString}
     chunk += `${splitText2Lines[i]}\n`;
   }
 
+  if (chunk) {
+    chunks.push(chunk);
+  }
+
   return {
     chunks,
     chars: chunks.reduce((sum, chunk) => sum + chunk.length, 0)

packages/global/common/string/time.ts

@@ -6,6 +6,42 @@ export const formatTime2YMDHM = (time?: Date) =>
 export const formatTime2YMD = (time?: Date) => (time ? dayjs(time).format('YYYY-MM-DD') : '');
 export const formatTime2HM = (time: Date = new Date()) => dayjs(time).format('HH:mm');
 
+/**
+ * 格式化时间成聊天格式
+ */
+export const formatTimeToChatTime = (time: Date) => {
+  const now = dayjs();
+  const target = dayjs(time);
+
+  // 如果传入时间小于60秒，返回刚刚
+  if (now.diff(target, 'second') < 60) {
+    return '刚刚';
+  }
+
+  // 如果时间是今天，展示几时:几分
+  if (now.isSame(target, 'day')) {
+    return target.format('HH:mm');
+  }
+
+  // 如果是昨天，展示昨天
+  if (now.subtract(1, 'day').isSame(target, 'day')) {
+    return '昨天';
+  }
+
+  // 如果是前天，展示前天
+  if (now.subtract(2, 'day').isSame(target, 'day')) {
+    return '前天';
+  }
+
+  // 如果是今年，展示某月某日
+  if (now.isSame(target, 'year')) {
+    return target.format('MM/DD');
+  }
+
+  // 如果是更久之前，展示某年某月某日
+  return target.format('YYYY/M/D');
+};
+
 /* cron time parse */
 export const cronParser2Fields = (cronString: string) => {
   try {

packages/global/common/system/types/index.d.ts

@@ -66,6 +66,8 @@ export type SystemEnvType = {
   vectorMaxProcess: number;
   qaMaxProcess: number;
   pgHNSWEfSearch: number;
+  tokenWorkers: number; // token count max worker
+
   oneapiUrl?: string;
   chatApiKey?: string;
 };

packages/global/core/app/constants.ts

@@ -1,4 +1,4 @@
-import { AppWhisperConfigType } from './type';
+import { AppTTSConfigType, AppWhisperConfigType } from './type';
 
 export enum AppTypeEnum {
   simple = 'simple',
@@ -13,8 +13,16 @@ export const AppTypeMap = {
   }
 };
 
+export const defaultTTSConfig: AppTTSConfigType = { type: 'web' };
+
 export const defaultWhisperConfig: AppWhisperConfigType = {
   open: false,
   autoSend: false,
   autoTTSResponse: false
 };
+
+export const defaultChatInputGuideConfig = {
+  open: false,
+  textList: [],
+  customUrl: ''
+};

packages/global/core/app/type.d.ts

@@ -8,7 +8,7 @@ import { DatasetSearchModeEnum } from '../dataset/constants';
 import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
 import { StoreEdgeItemType } from '../workflow/type/edge';
 
-export interface AppSchema {
+export type AppSchema = {
   _id: string;
   teamId: string;
   tmbId: string;
@@ -23,13 +23,14 @@ export interface AppSchema {
   edges: StoreEdgeItemType[];
 
   // App system config
+  chatConfig: AppChatConfigType;
   scheduledTriggerConfig?: AppScheduledTriggerConfigType | null;
   scheduledTriggerNextTime?: Date;
 
   permission: `${PermissionTypeEnum}`;
   inited?: boolean;
   teamTags: string[];
-}
+};
 
 export type AppListItemType = {
   _id: string;
@@ -66,32 +67,19 @@ export type AppSimpleEditFormType = {
     datasetSearchExtensionBg?: string;
   };
   selectedTools: FlowNodeTemplateType[];
-  userGuide: {
+  chatConfig: AppChatConfigType;
-    welcomeText: string;
-    variables: {
-      id: string;
-      key: string;
-      label: string;
-      type: `${VariableInputEnum}`;
-      required: boolean;
-      maxLen: number;
-      enums: {
-        value: string;
-      }[];
-    }[];
-    questionGuide: boolean;
-    tts: {
-      type: 'none' | 'web' | 'model';
-      model?: string | undefined;
-      voice?: string | undefined;
-      speed?: number | undefined;
-    };
-    whisper: AppWhisperConfigType;
-    scheduleTrigger: AppScheduledTriggerConfigType | null;
-  };
 };
 
-/* app function config */
+/* app chat config type */
+export type AppChatConfigType = {
+  welcomeText?: string;
+  variables?: VariableItemType[];
+  questionGuide?: boolean;
+  ttsConfig?: AppTTSConfigType;
+  whisperConfig?: AppWhisperConfigType;
+  scheduledTriggerConfig?: AppScheduledTriggerConfigType;
+  chatInputGuide?: ChatInputGuideConfigType;
+};
 export type SettingAIDataType = {
   model: string;
   temperature: number;
@@ -123,6 +111,11 @@ export type AppWhisperConfigType = {
   autoSend: boolean;
   autoTTSResponse: boolean;
 };
+// question guide text
+export type ChatInputGuideConfigType = {
+  open: boolean;
+  customUrl: string;
+};
 // interval timer
 export type AppScheduledTriggerConfigType = {
   cronString: string;

packages/global/core/app/utils.ts

@@ -1,49 +1,42 @@
-import type { AppSimpleEditFormType } from '../app/type';
+import type { AppChatConfigType, AppSimpleEditFormType } from '../app/type';
 import { FlowNodeTypeEnum } from '../workflow/node/constant';
 import { NodeInputKeyEnum, FlowNodeTemplateTypeEnum } from '../workflow/constants';
 import type { FlowNodeInputItemType } from '../workflow/type/io.d';
-import { getGuideModule, splitGuideModule } from '../workflow/utils';
+import { getAppChatConfig } from '../workflow/utils';
 import { StoreNodeItemType } from '../workflow/type';
 import { DatasetSearchModeEnum } from '../dataset/constants';
-import { defaultWhisperConfig } from './constants';
 
-export const getDefaultAppForm = (): AppSimpleEditFormType => {
+export const getDefaultAppForm = (): AppSimpleEditFormType => ({
-  return {
+  aiSettings: {
-    aiSettings: {
+    model: 'gpt-3.5-turbo',
-      model: 'gpt-3.5-turbo',
+    systemPrompt: '',
-      systemPrompt: '',
+    temperature: 0,
-      temperature: 0,
+    isResponseAnswerText: true,
-      isResponseAnswerText: true,
+    maxHistories: 6,
-      maxHistories: 6,
+    maxToken: 4000
-      maxToken: 4000
+  },
-    },
+  dataset: {
-    dataset: {
+    datasets: [],
-      datasets: [],
+    similarity: 0.4,
-      similarity: 0.4,
+    limit: 1500,
-      limit: 1500,
+    searchMode: DatasetSearchModeEnum.embedding,
-      searchMode: DatasetSearchModeEnum.embedding,
+    usingReRank: false,
-      usingReRank: false,
+    datasetSearchUsingExtensionQuery: true,
-      datasetSearchUsingExtensionQuery: true,
+    datasetSearchExtensionBg: ''
-      datasetSearchExtensionBg: ''
+  },
-    },
+  selectedTools: [],
-    selectedTools: [],
+  chatConfig: {}
-    userGuide: {
+});
-      welcomeText: '',
-      variables: [],
-      questionGuide: false,
-      tts: {
-        type: 'web'
-      },
-      whisper: defaultWhisperConfig,
-      scheduleTrigger: null
-    }
-  };
-};
 
 /* format app nodes to edit form */
-export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => {
+export const appWorkflow2Form = ({
+  nodes,
+  chatConfig
+}: {
+  nodes: StoreNodeItemType[];
+  chatConfig: AppChatConfigType;
+}) => {
   const defaultAppForm = getDefaultAppForm();
-
   const findInputValueByKey = (inputs: FlowNodeInputItemType[], key: string) => {
     return inputs.find((item) => item.key === key)?.value;
   };
@@ -102,24 +95,6 @@ export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => {
         node.inputs,
         NodeInputKeyEnum.datasetSearchExtensionBg
       );
-    } else if (node.flowNodeType === FlowNodeTypeEnum.systemConfig) {
-      const {
-        welcomeText,
-        variableNodes,
-        questionGuide,
-        ttsConfig,
-        whisperConfig,
-        scheduledTriggerConfig
-      } = splitGuideModule(getGuideModule(nodes));
-
-      defaultAppForm.userGuide = {
-        welcomeText: welcomeText,
-        variables: variableNodes,
-        questionGuide: questionGuide,
-        tts: ttsConfig,
-        whisper: whisperConfig,
-        scheduleTrigger: scheduledTriggerConfig
-      };
     } else if (node.flowNodeType === FlowNodeTypeEnum.pluginModule) {
       if (!node.pluginId) return;
 
@@ -131,10 +106,17 @@ export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => {
         intro: node.intro || '',
         flowNodeType: node.flowNodeType,
         showStatus: node.showStatus,
+        version: '481',
         inputs: node.inputs,
         outputs: node.outputs,
         templateType: FlowNodeTemplateTypeEnum.other
       });
+    } else if (node.flowNodeType === FlowNodeTypeEnum.systemConfig) {
+      defaultAppForm.chatConfig = getAppChatConfig({
+        chatConfig,
+        systemConfigNode: node,
+        isPublicFetch: true
+      });
     }
   });
 

packages/global/core/app/version.d.ts

@@ -1,5 +1,6 @@
 import { StoreNodeItemType } from '../workflow/type';
 import { StoreEdgeItemType } from '../workflow/type/edge';
+import { AppChatConfigType } from './type';
 
 export type AppVersionSchemaType = {
   _id: string;
@@ -7,4 +8,5 @@ export type AppVersionSchemaType = {
   time: Date;
   nodes: StoreNodeItemType[];
   edges: StoreEdgeItemType[];
+  chatConfig: AppChatConfigType;
 };

packages/global/core/chat/inputGuide/type.d.ts

@@ -0,0 +1,5 @@
+export type ChatInputGuideSchemaType = {
+  _id: string;
+  appId: string;
+  text: string;
+};

packages/global/core/chat/type.d.ts

@@ -10,7 +10,7 @@ import {
 import { FlowNodeTypeEnum } from '../workflow/node/constant';
 import { NodeOutputKeyEnum } from '../workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '../workflow/runtime/constants';
-import { AppSchema, VariableItemType } from '../app/type';
+import { AppChatConfigType, AppSchema, VariableItemType } from '../app/type';
 import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
 import { DatasetSearchModeEnum } from '../dataset/constants';
 import { ChatBoxInputType } from '../../../../projects/app/src/components/ChatBox/type';
@@ -139,7 +139,7 @@ export type ChatHistoryItemType = HistoryItemType & {
 /* ------- response data ------------ */
 export type ChatHistoryItemResType = DispatchNodeResponseType & {
   nodeId: string;
-  moduleType: `${FlowNodeTypeEnum}`;
+  moduleType: FlowNodeTypeEnum;
   moduleName: string;
 };
 

packages/global/core/dataset/api.d.ts

@@ -11,31 +11,42 @@ export type DatasetUpdateBody = {
   intro?: string;
   permission?: DatasetSchemaType['permission'];
   agentModel?: LLMModelItemType;
-  websiteConfig?: DatasetSchemaType['websiteConfig'];
   status?: DatasetSchemaType['status'];
+
+  websiteConfig?: DatasetSchemaType['websiteConfig'];
+  externalReadUrl?: DatasetSchemaType['externalReadUrl'];
 };
 
 /* ================= collection ===================== */
 export type DatasetCollectionChunkMetadataType = {
   parentId?: string;
-  trainingType?: `${TrainingModeEnum}`;
+  trainingType?: TrainingModeEnum;
   chunkSize?: number;
   chunkSplitter?: string;
   qaPrompt?: string;
   metadata?: Record<string, any>;
 };
+
+// create collection params
 export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
   datasetId: string;
   name: string;
-  type: `${DatasetCollectionTypeEnum}`;
+  type: DatasetCollectionTypeEnum;
+
+  tags?: string[];
+
   fileId?: string;
   rawLink?: string;
+  externalFileId?: string;
+
+  externalFileUrl?: string;
   rawTextLength?: number;
   hashRawText?: string;
 };
 
 export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & {
   datasetId: string;
+  tags?: string[];
 };
 export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
   name: string;
@@ -56,6 +67,11 @@ export type CsvTableCreateDatasetCollectionParams = {
   parentId?: string;
   fileId: string;
 };
+export type ExternalFileCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & {
+  externalFileId?: string;
+  externalFileUrl: string;
+  filename?: string;
+};
 
 /* ================= data ===================== */
 export type PgSearchRawType = {
@@ -78,7 +94,7 @@ export type PostWebsiteSyncParams = {
 export type PushDatasetDataProps = {
   collectionId: string;
   data: PushDatasetDataChunkProps[];
-  trainingMode: `${TrainingModeEnum}`;
+  trainingMode: TrainingModeEnum;
   prompt?: string;
   billId?: string;
 };

packages/global/core/dataset/collection/constants.ts

@@ -0,0 +1,6 @@
+/* sourceId = prefix-id; id=fileId;link url;externalFileId */
+export enum CollectionSourcePrefixEnum {
+  local = 'local',
+  link = 'link',
+  external = 'external'
+}

packages/global/core/dataset/collection/utils.ts

@@ -0,0 +1,14 @@
+import { CollectionWithDatasetType, DatasetCollectionSchemaType } from '../type';
+
+export const getCollectionSourceData = (
+  collection?: CollectionWithDatasetType | DatasetCollectionSchemaType
+) => {
+  return {
+    sourceId:
+      collection?.fileId ||
+      collection?.rawLink ||
+      collection?.externalFileId ||
+      collection?.externalFileUrl,
+    sourceName: collection?.name || ''
+  };
+};

packages/global/core/dataset/constants.ts

@@ -2,23 +2,29 @@
 export enum DatasetTypeEnum {
   folder = 'folder',
   dataset = 'dataset',
-  websiteDataset = 'websiteDataset' // depp link
+  websiteDataset = 'websiteDataset', // depp link
+  externalFile = 'externalFile'
 }
 export const DatasetTypeMap = {
   [DatasetTypeEnum.folder]: {
     icon: 'common/folderFill',
-    label: 'core.dataset.Folder Dataset',
+    label: 'Folder Dataset',
     collectionLabel: 'common.Folder'
   },
   [DatasetTypeEnum.dataset]: {
     icon: 'core/dataset/commonDataset',
-    label: 'core.dataset.Common Dataset',
+    label: 'Common Dataset',
     collectionLabel: 'common.File'
   },
   [DatasetTypeEnum.websiteDataset]: {
     icon: 'core/dataset/websiteDataset',
-    label: 'core.dataset.Website Dataset',
+    label: 'Website Dataset',
     collectionLabel: 'common.Website'
+  },
+  [DatasetTypeEnum.externalFile]: {
+    icon: 'core/dataset/externalDataset',
+    label: 'External File',
+    collectionLabel: 'common.File'
   }
 };
 
@@ -38,9 +44,11 @@ export const DatasetStatusMap = {
 /* ------------ collection -------------- */
 export enum DatasetCollectionTypeEnum {
   folder = 'folder',
+  virtual = 'virtual',
+
   file = 'file',
   link = 'link', // one link
-  virtual = 'virtual'
+  externalFile = 'externalFile'
 }
 export const DatasetCollectionTypeMap = {
   [DatasetCollectionTypeEnum.folder]: {
@@ -49,6 +57,9 @@ export const DatasetCollectionTypeMap = {
   [DatasetCollectionTypeEnum.file]: {
     name: 'core.dataset.file'
   },
+  [DatasetCollectionTypeEnum.externalFile]: {
+    name: 'core.dataset.externalFile'
+  },
   [DatasetCollectionTypeEnum.link]: {
     name: 'core.dataset.link'
   },
@@ -77,7 +88,8 @@ export enum ImportDataSourceEnum {
   fileLocal = 'fileLocal',
   fileLink = 'fileLink',
   fileCustom = 'fileCustom',
-  csvTable = 'csvTable'
+  csvTable = 'csvTable',
+  externalFile = 'externalFile'
 }
 
 export enum TrainingModeEnum {
@@ -163,3 +175,10 @@ export const SearchScoreTypeMap = {
 
 export const CustomCollectionIcon = 'common/linkBlue';
 export const LinkCollectionIcon = 'common/linkBlue';
+
+/* source prefix */
+export enum DatasetSourceReadTypeEnum {
+  fileLocal = 'fileLocal',
+  link = 'link',
+  externalFile = 'externalFile'
+}

packages/global/core/dataset/read.ts

@@ -0,0 +1,14 @@
+import { DatasetSourceReadTypeEnum, ImportDataSourceEnum } from './constants';
+
+export const importType2ReadType = (type: ImportDataSourceEnum) => {
+  if (type === ImportDataSourceEnum.csvTable || type === ImportDataSourceEnum.fileLocal) {
+    return DatasetSourceReadTypeEnum.fileLocal;
+  }
+  if (type === ImportDataSourceEnum.fileLink) {
+    return DatasetSourceReadTypeEnum.link;
+  }
+  if (type === ImportDataSourceEnum.externalFile) {
+    return DatasetSourceReadTypeEnum.externalFile;
+  }
+  return DatasetSourceReadTypeEnum.link;
+};

packages/global/core/dataset/type.d.ts

@@ -22,13 +22,16 @@ export type DatasetSchemaType = {
   vectorModel: string;
   agentModel: string;
   intro: string;
-  type: `${DatasetTypeEnum}`;
+  type: DatasetTypeEnum;
   status: `${DatasetStatusEnum}`;
   permission: `${PermissionTypeEnum}`;
+
+  // metadata
   websiteConfig?: {
     url: string;
     selector: string;
   };
+  externalReadUrl?: string;
 };
 
 export type DatasetCollectionSchemaType = {
@@ -38,20 +41,24 @@ export type DatasetCollectionSchemaType = {
   datasetId: string;
   parentId?: string;
   name: string;
-  type: `${DatasetCollectionTypeEnum}`;
+  type: DatasetCollectionTypeEnum;
   createTime: Date;
   updateTime: Date;
 
-  trainingType: `${TrainingModeEnum}`;
+  trainingType: TrainingModeEnum;
   chunkSize: number;
   chunkSplitter?: string;
   qaPrompt?: string;
 
-  fileId?: string;
+  tags?: string[];
-  rawLink?: string;
+
+  fileId?: string; // local file id
+  rawLink?: string; // link url
+  externalFileId?: string; //external file id
 
   rawTextLength?: number;
   hashRawText?: string;
+  externalFileUrl?: string; // external import url
   metadata?: {
     webPageSelector?: string;
     relatedImgId?: string; // The id of the associated image collections
@@ -80,6 +87,7 @@ export type DatasetDataSchemaType = {
   a: string; // answer or custom content
   fullTextToken: string;
   indexes: DatasetDataIndexItemType[];
+  rebuilding?: boolean;
 };
 
 export type DatasetTrainingSchemaType = {
@@ -92,9 +100,10 @@ export type DatasetTrainingSchemaType = {
   billId: string;
   expireAt: Date;
   lockTime: Date;
-  mode: `${TrainingModeEnum}`;
+  mode: TrainingModeEnum;
   model: string;
   prompt: string;
+  dataId?: string;
   q: string;
   a: string;
   chunkIndex: number;
@@ -110,13 +119,19 @@ export type DatasetDataWithCollectionType = Omit<DatasetDataSchemaType, 'collect
 };
 
 /* ================= dataset ===================== */
+export type DatasetSimpleItemType = {
+  _id: string;
+  avatar: string;
+  name: string;
+  vectorModel: VectorModelItemType;
+};
 export type DatasetListItemType = {
   _id: string;
   parentId: string;
   avatar: string;
   name: string;
   intro: string;
-  type: `${DatasetTypeEnum}`;
+  type: DatasetTypeEnum;
   isOwner: boolean;
   canWrite: boolean;
   permission: `${PermissionTypeEnum}`;

packages/global/core/dataset/utils.ts

@@ -3,7 +3,7 @@ import { getFileIcon } from '../../common/file/icon';
 import { strIsLink } from '../../common/string/tools';
 
 export function getCollectionIcon(
-  type: `${DatasetCollectionTypeEnum}` = DatasetCollectionTypeEnum.file,
+  type: DatasetCollectionTypeEnum = DatasetCollectionTypeEnum.file,
   name = ''
 ) {
   if (type === DatasetCollectionTypeEnum.folder) {
@@ -24,13 +24,13 @@ export function getSourceNameIcon({
   sourceName: string;
   sourceId?: string;
 }) {
-  if (strIsLink(sourceId)) {
+  const fileIcon = getFileIcon(decodeURIComponent(sourceName), '');
-    return 'common/linkBlue';
-  }
-  const fileIcon = getFileIcon(sourceName, '');
   if (fileIcon) {
     return fileIcon;
   }
+  if (strIsLink(sourceId)) {
+    return 'common/linkBlue';
+  }
 
   return 'file/fill/manual';
 }
@@ -46,7 +46,7 @@ export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: strin
   };
 }
 
-export const predictDataLimitLength = (mode: `${TrainingModeEnum}`, data: any[]) => {
+export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => {
   if (mode === TrainingModeEnum.qa) return data.length * 20;
   if (mode === TrainingModeEnum.auto) return data.length * 5;
   return data.length;

packages/global/core/plugin/httpPlugin/utils.ts

@@ -282,6 +282,7 @@ export const httpApiSchema2Plugins = async ({
           x: 616.4226348688949,
           y: -165.05298493910115
         },
+        version: PluginInputModule.version,
         inputs: pluginInputs,
         outputs: pluginOutputs
       },
@@ -296,6 +297,7 @@ export const httpApiSchema2Plugins = async ({
           x: 1607.7142331269126,
           y: -151.8669210746189
         },
+        version: PluginOutputModule.version,
         inputs: [
           {
             key: pluginOutputKey,
@@ -334,6 +336,7 @@ export const httpApiSchema2Plugins = async ({
           x: 1042.549746602742,
           y: -447.77496332641647
         },
+        version: HttpModule468.version,
         inputs: [
           {
             key: NodeInputKeyEnum.addInputParam,

packages/global/core/plugin/type.d.ts

@@ -23,6 +23,7 @@ export type PluginItemSchema = {
     customHeaders?: string;
   };
   version?: 'v1' | 'v2';
+  nodeVersion?: string;
 };
 
 /* plugin template */
@@ -32,6 +33,7 @@ export type PluginTemplateType = PluginRuntimeType & {
   source: `${PluginSourceEnum}`;
   templateType: FlowNodeTemplateType['templateType'];
   intro: string;
+  nodeVersion: string;
 };
 
 export type PluginRuntimeType = {

packages/global/core/workflow/api.d.ts

@@ -1,7 +1,7 @@
 import { VectorModelItemType } from '../ai/model.d';
 import { NodeInputKeyEnum } from './constants';
 
-export type SelectedDatasetType = { datasetId: string; vectorModel: VectorModelItemType }[];
+export type SelectedDatasetType = { datasetId: string }[];
 
 export type HttpBodyType<T = Record<string, any>> = {
   [NodeInputKeyEnum.addInputParam]: Record<string, any>;

packages/global/core/workflow/constants.ts

@@ -45,6 +45,7 @@ export enum NodeInputKeyEnum {
   whisper = 'whisper',
   variables = 'variables',
   scheduleTrigger = 'scheduleTrigger',
+  chatInputGuide = 'chatInputGuide',
 
   // entry
   userChatInput = 'userChatInput',

packages/global/core/workflow/runtime/utils.ts

@@ -1,5 +1,5 @@
 import { ChatCompletionRequestMessageRoleEnum } from '../../ai/constants';
-import { NodeOutputKeyEnum } from '../constants';
+import { NodeInputKeyEnum, NodeOutputKeyEnum } from '../constants';
 import { FlowNodeTypeEnum } from '../node/constant';
 import { StoreNodeItemType } from '../type';
 import { StoreEdgeItemType } from '../type/edge';
@@ -8,6 +8,23 @@ import { VARIABLE_NODE_ID } from '../constants';
 import { isReferenceValue } from '../utils';
 import { ReferenceValueProps } from '../type/io';
 
+export const getMaxHistoryLimitFromNodes = (nodes: StoreNodeItemType[]): number => {
+  let limit = 10;
+  nodes.forEach((node) => {
+    node.inputs.forEach((input) => {
+      if (
+        (input.key === NodeInputKeyEnum.history ||
+          input.key === NodeInputKeyEnum.historyMaxAmount) &&
+        typeof input.value === 'number'
+      ) {
+        limit = Math.max(limit, input.value);
+      }
+    });
+  });
+
+  return limit * 2;
+};
+
 export const initWorkflowEdgeStatus = (edges: StoreEdgeItemType[]): RuntimeEdgeItemType[] => {
   return (
     edges?.map((edge) => ({

packages/global/core/workflow/template/system/aiChat.ts

@@ -31,6 +31,7 @@ export const AiChatModule: FlowNodeTemplateType = {
   intro: 'AI 大模型对话',
   showStatus: true,
   isTool: true,
+  version: '481',
   inputs: [
     Input_Template_SettingAiModel,
     // --- settings modal

packages/global/core/workflow/template/system/assignedAnswer.ts

@@ -17,6 +17,8 @@ export const AssignedAnswerModule: FlowNodeTemplateType = {
   name: '指定回复',
   intro:
     '该模块可以直接回复一段指定的内容。常用于引导、提示。非字符串内容传入时，会转成字符串进行输出。',
+  version: '481',
+  isTool: true,
   inputs: [
     {
       key: NodeInputKeyEnum.answerText,

packages/global/core/workflow/template/system/classifyQuestion.ts

@@ -29,6 +29,7 @@ export const ClassifyQuestionModule: FlowNodeTemplateType = {
   name: '问题分类',
   intro: `根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：\n类型1: 打招呼\n类型2: 关于商品“使用”问题\n类型3: 关于商品“购买”问题\n类型4: 其他问题`,
   showStatus: true,
+  version: '481',
   inputs: [
     {
       ...Input_Template_SelectAIModel,

packages/global/core/workflow/template/system/contextExtract.ts

@@ -25,6 +25,7 @@ export const ContextExtractModule: FlowNodeTemplateType = {
   intro: '可从文本中提取指定的数据，例如：sql语句、搜索关键词、代码等',
   showStatus: true,
   isTool: true,
+  version: '481',
   inputs: [
     {
       ...Input_Template_SelectAIModel,

packages/global/core/workflow/template/system/datasetConcat.ts

@@ -42,6 +42,7 @@ export const DatasetConcatModule: FlowNodeTemplateType = {
   name: '知识库搜索引用合并',
   intro: '可以将多个知识库搜索结果进行合并输出。使用 RRF 的合并方式进行最终排序输出。',
   showStatus: false,
+  version: '481',
   inputs: [
     {
       key: NodeInputKeyEnum.datasetMaxTokens,

packages/global/core/workflow/template/system/datasetSearch.ts

@@ -28,6 +28,7 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
   intro: Dataset_SEARCH_DESC,
   showStatus: true,
   isTool: true,
+  version: '481',
   inputs: [
     {
       key: NodeInputKeyEnum.datasetSelectList,
@@ -35,7 +36,6 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
       label: 'core.module.input.label.Select dataset',
       value: [],
       valueType: WorkflowIOValueTypeEnum.selectDataset,
-      list: [],
       required: true
     },
     {

packages/global/core/workflow/template/system/emptyNode.ts

@@ -12,6 +12,7 @@ export const EmptyNode: FlowNodeTemplateType = {
   avatar: '',
   name: '',
   intro: '',
+  version: '481',
   inputs: [],
   outputs: []
 };

packages/global/core/workflow/template/system/globalVariable.ts

@@ -1,32 +0,0 @@
-import { FlowNodeTemplateTypeEnum, WorkflowIOValueTypeEnum } from '../../constants';
-import { getHandleConfig } from '../utils';
-import { FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
-import { VariableItemType } from '../../../app/type';
-import { FlowNodeTemplateType } from '../../type';
-
-export const getGlobalVariableNode = ({
-  id,
-  variables
-}: {
-  id: string;
-  variables: VariableItemType[];
-}): FlowNodeTemplateType => {
-  return {
-    id,
-    templateType: FlowNodeTemplateTypeEnum.other,
-    flowNodeType: FlowNodeTypeEnum.systemConfig,
-    sourceHandle: getHandleConfig(true, true, true, true),
-    targetHandle: getHandleConfig(true, true, true, true),
-    avatar: '/imgs/workflow/variable.png',
-    name: '全局变量',
-    intro: '',
-    inputs: [],
-    outputs: variables.map((item) => ({
-      id: item.key,
-      key: item.key,
-      valueType: WorkflowIOValueTypeEnum.string,
-      type: FlowNodeOutputTypeEnum.static,
-      label: item.label
-    }))
-  };
-};

packages/global/core/workflow/template/system/http468.ts

@@ -25,6 +25,7 @@ export const HttpModule468: FlowNodeTemplateType = {
   intro: '可以发出一个 HTTP 请求，实现更为复杂的操作（联网搜索、数据库查询等）',
   showStatus: true,
   isTool: true,
+  version: '481',
   inputs: [
     {
       ...Input_Template_DynamicInput,

packages/global/core/workflow/template/system/ifElse/index.ts

@@ -22,6 +22,7 @@ export const IfElseNode: FlowNodeTemplateType = {
   name: '判断器',
   intro: '根据一定的条件，执行不同的分支。',
   showStatus: true,
+  version: '481',
   inputs: [
     {
       key: NodeInputKeyEnum.ifElseList,

packages/global/core/workflow/template/system/laf.ts

@@ -25,6 +25,7 @@ export const LafModule: FlowNodeTemplateType = {
   intro: '可以调用Laf账号下的云函数。',
   showStatus: true,
   isTool: true,
+  version: '481',
   inputs: [
     {
       ...Input_Template_DynamicInput,

packages/global/core/workflow/template/system/pluginInput.ts

@@ -15,6 +15,7 @@ export const PluginInputModule: FlowNodeTemplateType = {
   name: '自定义插件输入',
   intro: '自定义配置外部输入，使用插件时，仅暴露自定义配置的输入',
   showStatus: false,
+  version: '481',
   inputs: [],
   outputs: []
 };

packages/global/core/workflow/template/system/pluginOutput.ts

@@ -15,6 +15,7 @@ export const PluginOutputModule: FlowNodeTemplateType = {
   name: '自定义插件输出',
   intro: '自定义配置外部输出，使用插件时，仅暴露自定义配置的输出',
   showStatus: false,
+  version: '481',
   inputs: [],
   outputs: []
 };

packages/global/core/workflow/template/system/queryExtension.ts

@@ -29,6 +29,7 @@ export const AiQueryExtension: FlowNodeTemplateType = {
   intro:
     '使用问题优化功能，可以提高知识库连续对话时搜索的精度。使用该功能后，会先利用 AI 根据上下文构建一个或多个新的检索词，这些检索词更利于进行知识库搜索。该模块已内置在知识库搜索模块中，如果您仅进行一次知识库搜索，可直接使用知识库内置的补全功能。',
   showStatus: true,
+  version: '481',
   inputs: [
     {
       ...Input_Template_SelectAIModel,

packages/global/core/workflow/template/system/runApp.ts

@@ -23,6 +23,7 @@ export const RunAppModule: FlowNodeTemplateType = {
   name: '应用调用',
   intro: '可以选择一个其他应用进行调用',
   showStatus: true,
+  version: '481',
   inputs: [
     {
       key: NodeInputKeyEnum.runAppSelectApp,

packages/global/core/workflow/template/system/runPlugin.ts

@@ -13,6 +13,7 @@ export const RunPluginModule: FlowNodeTemplateType = {
   name: '',
   showStatus: false,
   isTool: true,
+  version: '481',
   inputs: [], // [{key:'pluginId'},...]
   outputs: []
 };

packages/global/core/workflow/template/system/stopTool.ts

@@ -13,6 +13,7 @@ export const StopToolNode: FlowNodeTemplateType = {
   name: '工具调用终止',
   intro:
     '该模块需配置工具调用使用。当该模块被执行时，本次工具调用将会强制结束，并且不再调用AI针对工具调用结果回答问题。',
+  version: '481',
   inputs: [],
   outputs: []
 };

packages/global/core/workflow/template/system/systemConfig.ts

@@ -1,10 +1,6 @@
-import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
+import { FlowNodeTypeEnum } from '../../node/constant';
 import { FlowNodeTemplateType } from '../../type/index.d';
-import {
+import { FlowNodeTemplateTypeEnum, WorkflowIOValueTypeEnum } from '../../constants';
-  WorkflowIOValueTypeEnum,
-  NodeInputKeyEnum,
-  FlowNodeTemplateTypeEnum
-} from '../../constants';
 import { getHandleConfig } from '../utils';
 
 export const SystemConfigNode: FlowNodeTemplateType = {
@@ -18,44 +14,7 @@ export const SystemConfigNode: FlowNodeTemplateType = {
   intro: '可以配置应用的系统参数。',
   unique: true,
   forbidDelete: true,
-  inputs: [
+  version: '481',
-    {
+  inputs: [],
-      key: NodeInputKeyEnum.welcomeText,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.string,
-      label: 'core.app.Welcome Text'
-    },
-    {
-      key: NodeInputKeyEnum.variables,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.any,
-      label: 'core.module.Variable',
-      value: []
-    },
-    {
-      key: NodeInputKeyEnum.questionGuide,
-      valueType: WorkflowIOValueTypeEnum.boolean,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      label: ''
-    },
-    {
-      key: NodeInputKeyEnum.tts,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.any,
-      label: ''
-    },
-    {
-      key: NodeInputKeyEnum.whisper,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.any,
-      label: ''
-    },
-    {
-      key: NodeInputKeyEnum.scheduleTrigger,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.any,
-      label: ''
-    }
-  ],
   outputs: []
 };

packages/global/core/workflow/template/system/tools.ts

@@ -30,6 +30,7 @@ export const ToolModule: FlowNodeTemplateType = {
   name: '工具调用(实验)',
   intro: '通过AI模型自动选择一个或多个功能块进行调用，也可以对插件进行调用。',
   showStatus: true,
+  version: '481',
   inputs: [
     {
       ...Input_Template_SettingAiModel,

packages/global/core/workflow/template/system/variableUpdate/index.tsx

@@ -18,6 +18,7 @@ export const VariableUpdateNode: FlowNodeTemplateType = {
   intro: '可以更新指定节点的输出值或更新全局变量',
   showStatus: true,
   isTool: false,
+  version: '481',
   inputs: [
     {
       key: NodeInputKeyEnum.updateList,

packages/global/core/workflow/template/system/workflowStart.ts

@@ -19,6 +19,7 @@ export const WorkflowStart: FlowNodeTemplateType = {
   intro: '',
   forbidDelete: true,
   unique: true,
+  version: '481',
   inputs: [{ ...Input_Template_UserChatInput, toolDescription: '用户问题' }],
   outputs: [
     {

packages/global/core/workflow/type/index.d.ts

@@ -22,12 +22,13 @@ import { RuntimeEdgeItemType, StoreEdgeItemType } from './edge';
 import { NextApiResponse } from 'next';
 
 export type FlowNodeCommonType = {
-  flowNodeType: `${FlowNodeTypeEnum}`; // render node card
+  flowNodeType: FlowNodeTypeEnum; // render node card
 
   avatar?: string;
   name: string;
   intro?: string; // template list intro
   showStatus?: boolean; // chatting response step status
+  version: string;
 
   // data
   inputs: FlowNodeInputItemType[];
@@ -63,6 +64,7 @@ export type FlowNodeTemplateType = FlowNodeCommonType & {
   // action
   forbidDelete?: boolean; // forbid delete
   unique?: boolean;
+  nodeVersion?: string;
 };
 export type FlowNodeItemType = FlowNodeTemplateType & {
   nodeId: string;

packages/global/core/workflow/utils.ts

@@ -1,4 +1,4 @@
-import { FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from './node/constant';
+import { FlowNodeInputTypeEnum, FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from './node/constant';
 import {
   WorkflowIOValueTypeEnum,
   NodeInputKeyEnum,
@@ -11,10 +11,16 @@ import type {
   VariableItemType,
   AppTTSConfigType,
   AppWhisperConfigType,
-  AppScheduledTriggerConfigType
+  AppScheduledTriggerConfigType,
+  ChatInputGuideConfigType,
+  AppChatConfigType
 } from '../app/type';
 import { EditorVariablePickerType } from '../../../web/components/common/Textarea/PromptEditor/type';
-import { defaultWhisperConfig } from '../app/constants';
+import {
+  defaultChatInputGuideConfig,
+  defaultTTSConfig,
+  defaultWhisperConfig
+} from '../app/constants';
 import { IfElseResultEnum } from './template/system/ifElse/constant';
 
 export const getHandleId = (nodeId: string, type: 'source' | 'target', key: string) => {
@@ -22,15 +28,9 @@ export const getHandleId = (nodeId: string, type: 'source' | 'target', key: stri
 };
 
 export const checkInputIsReference = (input: FlowNodeInputItemType) => {
-  const value = input.value;
+  if (input.renderTypeList?.[input?.selectedTypeIndex || 0] === FlowNodeInputTypeEnum.reference)
-  if (
-    Array.isArray(value) &&
-    value.length === 2 &&
-    typeof value[0] === 'string' &&
-    typeof value[1] === 'string'
-  ) {
     return true;
-  }
+
   return false;
 };
 
@@ -46,63 +46,78 @@ export const splitGuideModule = (guideModules?: StoreNodeItemType) => {
   const welcomeText: string =
     guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.welcomeText)?.value || '';
 
-  const variableNodes: VariableItemType[] =
+  const variables: VariableItemType[] =
     guideModules?.inputs.find((item) => item.key === NodeInputKeyEnum.variables)?.value || [];
 
   const questionGuide: boolean =
     !!guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.questionGuide)?.value ||
     false;
 
-  const ttsConfig: AppTTSConfigType = guideModules?.inputs?.find(
+  const ttsConfig: AppTTSConfigType =
-    (item) => item.key === NodeInputKeyEnum.tts
+    guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.tts)?.value ||
-  )?.value || { type: 'web' };
+    defaultTTSConfig;
 
   const whisperConfig: AppWhisperConfigType =
     guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.whisper)?.value ||
     defaultWhisperConfig;
 
-  const scheduledTriggerConfig: AppScheduledTriggerConfigType | null =
+  const scheduledTriggerConfig: AppScheduledTriggerConfigType = guideModules?.inputs?.find(
-    guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.scheduleTrigger)?.value ??
+    (item) => item.key === NodeInputKeyEnum.scheduleTrigger
-    null;
+  )?.value;
+
+  const chatInputGuide: ChatInputGuideConfigType =
+    guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.chatInputGuide)?.value ||
+    defaultChatInputGuideConfig;
 
   return {
     welcomeText,
-    variableNodes,
+    variables,
     questionGuide,
     ttsConfig,
     whisperConfig,
-    scheduledTriggerConfig
+    scheduledTriggerConfig,
+    chatInputGuide
   };
 };
-export const replaceAppChatConfig = ({
+export const getAppChatConfig = ({
-  node,
+  chatConfig,
-  variableList,
+  systemConfigNode,
-  welcomeText
+  storeVariables,
+  storeWelcomeText,
+  isPublicFetch = false
 }: {
-  node?: StoreNodeItemType;
+  chatConfig?: AppChatConfigType;
-  variableList?: VariableItemType[];
+  systemConfigNode?: StoreNodeItemType;
-  welcomeText?: string;
+  storeVariables?: VariableItemType[];
-}): StoreNodeItemType | undefined => {
+  storeWelcomeText?: string;
-  if (!node) return;
+  isPublicFetch: boolean;
-  return {
+}): AppChatConfigType => {
-    ...node,
+  const {
-    inputs: node.inputs.map((input) => {
+    welcomeText,
-      if (input.key === NodeInputKeyEnum.variables && variableList) {
+    variables,
-        return {
+    questionGuide,
-          ...input,
+    ttsConfig,
-          value: variableList
+    whisperConfig,
-        };
+    scheduledTriggerConfig,
-      }
+    chatInputGuide
-      if (input.key === NodeInputKeyEnum.welcomeText && welcomeText) {
+  } = splitGuideModule(systemConfigNode);
-        return {
-          ...input,
-          value: welcomeText
-        };
-      }
 
-      return input;
+  const config: AppChatConfigType = {
-    })
+    questionGuide,
+    ttsConfig,
+    whisperConfig,
+    scheduledTriggerConfig,
+    chatInputGuide,
+    ...chatConfig,
+    variables: storeVariables ?? chatConfig?.variables ?? variables,
+    welcomeText: storeWelcomeText ?? chatConfig?.welcomeText ?? welcomeText
   };
+
+  if (!isPublicFetch) {
+    config.scheduledTriggerConfig = undefined;
+  }
+
+  return config;
 };
 
 export const getOrInitModuleInputValue = (input: FlowNodeInputItemType) => {

packages/global/package.json

@@ -10,7 +10,7 @@
     "js-yaml": "^4.1.0",
     "jschardet": "3.1.1",
     "nanoid": "^4.0.1",
-    "next": "13.5.2",
+    "next": "14.2.3",
     "openai": "4.28.0",
     "openapi-types": "^12.1.3",
     "timezones-list": "^3.0.2"

packages/global/support/permission/constant.ts

@@ -20,3 +20,9 @@ export const PermissionTypeMap = {
     label: 'permission.Public'
   }
 };
+
+export enum ResourceTypeEnum {
+  team = 'team',
+  app = 'app',
+  dataset = 'dataset'
+}

packages/global/support/permission/type.d.ts

@@ -1,5 +1,7 @@
 import { AuthUserTypeEnum } from './constant';
 
+export type PermissionValueType = number;
+
 export type AuthResponseType = {
   teamId: string;
   tmbId: string;
@@ -9,3 +11,10 @@ export type AuthResponseType = {
   appId?: string;
   apikey?: string;
 };
+
+export type ResourcePermissionType = {
+  teamId: string;
+  tmbId: string;
+  resourceType: ResourceType;
+  permission: PermissionValueType;
+};

packages/global/support/user/team/constant.ts

@@ -1,6 +1,6 @@
 export const TeamCollectionName = 'teams';
-export const TeamMemberCollectionName = 'team.members';
+export const TeamMemberCollectionName = 'team_members';
-export const TeamTagsCollectionName = 'team.tags';
+export const TeamTagsCollectionName = 'team_tags';
 
 export enum TeamMemberRoleEnum {
   owner = 'owner',

packages/global/support/user/team/controller.d.ts

@@ -1,3 +1,4 @@
+import { PermissionValueType } from 'support/permission/type';
 import { TeamMemberRoleEnum } from './constant';
 import { LafAccountType, TeamMemberSchema } from './type';
 
@@ -44,3 +45,9 @@ export type InviteMemberResponse = Record<
   'invite' | 'inValid' | 'inTeam',
   { username: string; userId: string }[]
 >;
+
+export type UpdateTeamMemberPermissionProps = {
+  teamId: string;
+  memberIds: string[];
+  permission: PermissionValueType;
+};

packages/global/support/user/team/type.d.ts

@@ -1,6 +1,7 @@
 import type { UserModelSchema } from '../type';
 import type { TeamMemberRoleEnum, TeamMemberStatusEnum } from './constant';
 import { LafAccountType } from './type';
+import { PermissionValueType, ResourcePermissionType } from '../../permission/type';
 
 export type TeamSchema = {
   _id: string;
@@ -15,6 +16,7 @@ export type TeamSchema = {
     lastWebsiteSyncTime: Date;
   };
   lafAccount: LafAccountType;
+  defaultPermission: PermissionValueType;
 };
 export type tagsType = {
   label: string;
@@ -61,6 +63,7 @@ export type TeamItemType = {
   status: `${TeamMemberStatusEnum}`;
   canWrite: boolean;
   lafAccount?: LafAccountType;
+  defaultPermission: PermissionValueType;
 };
 
 export type TeamMemberItemType = {
@@ -69,8 +72,10 @@ export type TeamMemberItemType = {
   teamId: string;
   memberName: string;
   avatar: string;
+  // TODO: this should be deprecated.
   role: `${TeamMemberRoleEnum}`;
   status: `${TeamMemberStatusEnum}`;
+  permission: PermissionValueType;
 };
 
 export type TeamTagItemType = {

packages/service/common/buffer/rawText/schema.ts

@@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../mongo';
 const { Schema, model, models } = connectionMongo;
 import { RawTextBufferSchemaType } from './type';
 
-export const collectionName = 'buffer.rawText';
+export const collectionName = 'buffer_rawtexts';
 
 const RawTextBufferSchema = new Schema({
   sourceId: {

packages/service/common/buffer/tts/schema.ts

@@ -2,7 +2,7 @@ import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { TTSBufferSchemaType } from './type.d';
 
-export const collectionName = 'buffer.tts';
+export const collectionName = 'buffer_tts';
 
 const TTSBufferSchema = new Schema({
   bufferId: {

packages/service/common/file/gridfs/controller.ts

@@ -7,7 +7,7 @@ import { MongoFileSchema } from './schema';
 import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
-import { readFileRawContent } from '../read/utils';
+import { readRawContentByFileBuffer } from '../read/utils';
 import { PassThrough } from 'stream';
 
 export function getGFSCollection(bucket: `${BucketNameEnum}`) {
@@ -151,12 +151,12 @@ export const readFileContentFromMongo = async ({
   teamId,
   bucketName,
   fileId,
-  csvFormat = false
+  isQAImport = false
 }: {
   teamId: string;
   bucketName: `${BucketNameEnum}`;
   fileId: string;
-  csvFormat?: boolean;
+  isQAImport?: boolean;
 }): Promise<{
   rawText: string;
   filename: string;
@@ -196,9 +196,9 @@ export const readFileContentFromMongo = async ({
     });
   })();
 
-  const { rawText } = await readFileRawContent({
+  const { rawText } = await readRawContentByFileBuffer({
     extension,
-    csvFormat,
+    isQAImport,
     teamId,
     buffer: fileBuffers,
     encoding,

packages/service/common/file/read/utils.ts

@@ -1,9 +1,11 @@
-import { markdownProcess, simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
+import { markdownProcess } from '@fastgpt/global/common/string/markdown';
 import { uploadMongoImg } from '../image/controller';
 import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
 import { addHours } from 'date-fns';
 
 import { WorkerNameEnum, runWorker } from '../../../worker/utils';
+import fs from 'fs';
+import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
 import { ReadFileResponse } from '../../../worker/file/type';
 
 export const initMarkdownText = ({
@@ -27,42 +29,71 @@ export const initMarkdownText = ({
       })
   });
 
-export const readFileRawContent = async ({
+export type readRawTextByLocalFileParams = {
+  teamId: string;
+  path: string;
+  metadata?: Record<string, any>;
+};
+export const readRawTextByLocalFile = async (params: readRawTextByLocalFileParams) => {
+  const { path } = params;
+
+  const extension = path?.split('.')?.pop()?.toLowerCase() || '';
+
+  const buffer = fs.readFileSync(path);
+  const encoding = detectFileEncoding(buffer);
+
+  const { rawText } = await readRawContentByFileBuffer({
+    extension,
+    isQAImport: false,
+    teamId: params.teamId,
+    encoding,
+    buffer,
+    metadata: params.metadata
+  });
+
+  return {
+    rawText
+  };
+};
+
+export const readRawContentByFileBuffer = async ({
   extension,
-  csvFormat,
+  isQAImport,
   teamId,
   buffer,
   encoding,
   metadata
 }: {
-  csvFormat?: boolean;
+  isQAImport?: boolean;
   extension: string;
   teamId: string;
   buffer: Buffer;
   encoding: string;
   metadata?: Record<string, any>;
 }) => {
-  const result = await runWorker<ReadFileResponse>(WorkerNameEnum.readFile, {
+  let { rawText, formatText } = await runWorker<ReadFileResponse>(WorkerNameEnum.readFile, {
     extension,
-    csvFormat,
     encoding,
     buffer
   });
 
   // markdown data format
   if (['md', 'html', 'docx'].includes(extension)) {
-    result.rawText = await initMarkdownText({
+    rawText = await initMarkdownText({
       teamId: teamId,
-      md: result.rawText,
+      md: rawText,
       metadata: metadata
     });
   }
 
-  return result;
+  if (['csv', 'xlsx'].includes(extension)) {
-};
+    // qa data
+    if (isQAImport) {
+      rawText = rawText || '';
+    } else {
+      rawText = formatText || '';
+    }
+  }
 
-export const htmlToMarkdown = async (html?: string | null) => {
+  return { rawText };
-  const md = await runWorker<string>(WorkerNameEnum.htmlStr2Md, { html: html || '' });
-
-  return simpleMarkdownText(md);
 };

packages/service/common/middle/entry.ts

@@ -0,0 +1,44 @@
+import { jsonRes } from '../response';
+import type { NextApiResponse } from 'next';
+import { withNextCors } from './cors';
+import { ApiRequestProps } from '../../type/next';
+import { addLog } from '../system/log';
+
+export type NextApiHandler<T = any> = (
+  req: ApiRequestProps,
+  res: NextApiResponse<T>
+) => unknown | Promise<unknown>;
+
+export const NextEntry = ({ beforeCallback = [] }: { beforeCallback?: Promise<any>[] }) => {
+  return (...args: NextApiHandler[]): NextApiHandler => {
+    return async function api(req: ApiRequestProps, res: NextApiResponse) {
+      const start = Date.now();
+      addLog.info(`Request start ${req.url}`);
+      try {
+        await Promise.all([withNextCors(req, res), ...beforeCallback]);
+
+        let response = null;
+        for (const handler of args) {
+          response = await handler(req, res);
+        }
+
+        const contentType = res.getHeader('Content-Type');
+
+        addLog.info(`Request finish ${req.url}, time: ${Date.now() - start}ms`);
+
+        if ((!contentType || contentType === 'application/json') && !res.writableFinished) {
+          return jsonRes(res, {
+            code: 200,
+            data: response
+          });
+        }
+      } catch (error) {
+        return jsonRes(res, {
+          code: 500,
+          error,
+          url: req.url
+        });
+      }
+    };
+  };
+};

packages/service/common/mongo/sessionRun.ts

@@ -12,8 +12,6 @@ export const mongoSessionRun = async <T = unknown>(fn: (session: ClientSession)
 
     return result as T;
   } catch (error) {
-    console.log(error);
-
     await session.abortTransaction();
     await session.endSession();
     return Promise.reject(error);

packages/service/common/string/cheerio.ts

@@ -1,7 +1,7 @@
 import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api';
 import * as cheerio from 'cheerio';
 import axios from 'axios';
-import { htmlToMarkdown } from '../file/read/utils';
+import { htmlToMarkdown } from './utils';
 
 export const cheerioToHtml = ({
   fetchUrl,
@@ -77,9 +77,8 @@ export const urlsFetch = async ({
           $,
           selector
         });
-        console.log('html====', html);
+
         const md = await htmlToMarkdown(html);
-        console.log('html====', md);
 
         return {
           url,

packages/service/common/string/tiktoken/index.ts

@@ -12,27 +12,34 @@ import { getNanoid } from '@fastgpt/global/common/string/tools';
 import { addLog } from '../../system/log';
 
 export const getTiktokenWorker = () => {
-  if (global.tiktokenWorker) {
+  const maxWorkers = global.systemEnv?.tokenWorkers || 20;
-    return global.tiktokenWorker;
+
+  if (!global.tiktokenWorkers) {
+    global.tiktokenWorkers = [];
+  }
+
+  if (global.tiktokenWorkers.length >= maxWorkers) {
+    return global.tiktokenWorkers[Math.floor(Math.random() * global.tiktokenWorkers.length)];
   }
 
   const worker = getWorker(WorkerNameEnum.countGptMessagesTokens);
 
+  const i = global.tiktokenWorkers.push({
+    index: global.tiktokenWorkers.length,
+    worker,
+    callbackMap: {}
+  });
+
   worker.on('message', ({ id, data }: { id: string; data: number }) => {
-    const callback = global.tiktokenWorker?.callbackMap?.[id];
+    const callback = global.tiktokenWorkers[i - 1]?.callbackMap?.[id];
 
     if (callback) {
       callback?.(data);
-      delete global.tiktokenWorker.callbackMap[id];
+      delete global.tiktokenWorkers[i - 1].callbackMap[id];
     }
   });
 
-  global.tiktokenWorker = {
+  return global.tiktokenWorkers[i - 1];
-    worker,
-    callbackMap: {}
-  };
-
-  return global.tiktokenWorker;
 };
 
 export const countGptMessagesTokens = (
@@ -40,32 +47,46 @@ export const countGptMessagesTokens = (
   tools?: ChatCompletionTool[],
   functionCall?: ChatCompletionCreateParams.Function[]
 ) => {
-  return new Promise<number>((resolve) => {
+  return new Promise<number>(async (resolve) => {
-    const start = Date.now();
+    try {
+      const start = Date.now();
 
-    const { worker, callbackMap } = getTiktokenWorker();
+      const { worker, callbackMap } = getTiktokenWorker();
-    const id = getNanoid();
 
-    const timer = setTimeout(() => {
+      const id = getNanoid();
+
+      const timer = setTimeout(() => {
+        console.log('Count token Time out');
+        resolve(
+          messages.reduce((sum, item) => {
+            if (item.content) {
+              return sum + item.content.length * 0.5;
+            }
+            return sum;
+          }, 0)
+        );
+        delete callbackMap[id];
+      }, 60000);
+
+      callbackMap[id] = (data) => {
+        // 检测是否有内存泄漏
+        addLog.info(`Count token time: ${Date.now() - start}, token: ${data}`);
+        // console.log(process.memoryUsage());
+
+        resolve(data);
+        clearTimeout(timer);
+      };
+
+      // 可以进一步优化(传递100w token数据,实际需要300ms,较慢)
+      worker.postMessage({
+        id,
+        messages,
+        tools,
+        functionCall
+      });
+    } catch (error) {
       resolve(0);
-      delete callbackMap[id];
+    }
-    }, 300);
-
-    callbackMap[id] = (data) => {
-      resolve(data);
-      clearTimeout(timer);
-
-      // 检测是否有内存泄漏
-      // addLog.info(`Count token time: ${Date.now() - start}, token: ${data}`);
-      // console.log(process.memoryUsage());
-    };
-
-    worker.postMessage({
-      id,
-      messages,
-      tools,
-      functionCall
-    });
   });
 };
 

packages/service/common/string/utils.ts

@@ -0,0 +1,8 @@
+import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
+import { WorkerNameEnum, runWorker } from '../../worker/utils';
+
+export const htmlToMarkdown = async (html?: string | null) => {
+  const md = await runWorker<string>(WorkerNameEnum.htmlStr2Md, { html: html || '' });
+
+  return simpleMarkdownText(md);
+};

packages/service/common/system/tools.ts

@@ -20,3 +20,15 @@ export const initFastGPTConfig = (config?: FastGPTConfigFileType) => {
   global.whisperModel = config.whisperModel;
   global.reRankModels = config.reRankModels;
 };
+
+export const systemStartCb = () => {
+  process.on('uncaughtException', (err) => {
+    console.error('Uncaught Exception:', err);
+    // process.exit(1); // 退出进程
+  });
+
+  process.on('unhandledRejection', (reason, promise) => {
+    console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+    // process.exit(1); // 退出进程
+  });
+};

packages/service/common/vectorStore/pg/controller.ts

@@ -98,12 +98,15 @@ export const deleteDatasetDataVector = async (
       return `${teamIdWhere} ${datasetIdWhere}`;
     }
 
-    if ('idList' in props && props.idList) {
+    if ('idList' in props && Array.isArray(props.idList)) {
+      if (props.idList.length === 0) return;
       return `${teamIdWhere} id IN (${props.idList.map((id) => `'${String(id)}'`).join(',')})`;
     }
     return Promise.reject('deleteDatasetData: no where');
   })();
 
+  if (!where) return;
+
   try {
     await PgClient.delete(PgDatasetTableName, {
       where: [where]

packages/service/core/app/controller.ts

@@ -2,7 +2,7 @@ import { AppSchema } from '@fastgpt/global/core/app/type';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 import { getLLMModel } from '../ai/model';
-import { MongoAppVersion } from './versionSchema';
+import { MongoAppVersion } from './version/schema';
 
 export const beforeUpdateAppFormat = <T extends AppSchema['modules'] | undefined>({
   nodes
@@ -55,11 +55,13 @@ export const getAppLatestVersion = async (appId: string, app?: AppSchema) => {
   if (version) {
     return {
       nodes: version.nodes,
-      edges: version.edges
+      edges: version.edges,
+      chatConfig: version.chatConfig || app?.chatConfig || {}
     };
   }
   return {
     nodes: app?.modules || [],
-    edges: app?.edges || []
+    edges: app?.edges || [],
+    chatConfig: app?.chatConfig || {}
   };
 };

packages/service/core/app/schema.ts

@@ -10,6 +10,16 @@ import {
 
 export const AppCollectionName = 'apps';
 
+export const chatConfigType = {
+  welcomeText: String,
+  variables: Array,
+  questionGuide: Boolean,
+  ttsConfig: Object,
+  whisperConfig: Object,
+  scheduledTriggerConfig: Object,
+  chatInputGuide: Object
+};
+
 const AppSchema = new Schema({
   teamId: {
     type: Schema.Types.ObjectId,
@@ -47,6 +57,16 @@ const AppSchema = new Schema({
     default: () => new Date()
   },
 
+  // role and auth
+  permission: {
+    type: String,
+    enum: Object.keys(PermissionTypeMap),
+    default: PermissionTypeEnum.private
+  },
+  teamTags: {
+    type: [String]
+  },
+
   // tmp store
   modules: {
     type: Array,
@@ -56,6 +76,10 @@ const AppSchema = new Schema({
     type: Array,
     default: []
   },
+  chatConfig: {
+    type: chatConfigType,
+    default: {}
+  },
 
   scheduledTriggerConfig: {
     cronString: {
@@ -74,14 +98,6 @@ const AppSchema = new Schema({
 
   inited: {
     type: Boolean
-  },
-  permission: {
-    type: String,
-    enum: Object.keys(PermissionTypeMap),
-    default: PermissionTypeEnum.private
-  },
-  teamTags: {
-    type: [String]
   }
 });
 

packages/service/core/app/version/schema.ts

@@ -1,8 +1,9 @@
-import { connectionMongo, type Model } from '../../common/mongo';
+import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
+import { chatConfigType } from '../schema';
 
-export const AppVersionCollectionName = 'app.versions';
+export const AppVersionCollectionName = 'app_versions';
 
 const AppVersionSchema = new Schema({
   appId: {
@@ -21,6 +22,10 @@ const AppVersionSchema = new Schema({
   edges: {
     type: Array,
     default: []
+  },
+  chatConfig: {
+    type: chatConfigType,
+    default: {}
   }
 });
 

packages/service/core/chat/controller.ts

@@ -32,7 +32,7 @@ export async function getChatItems({
 
   return { history };
 }
-/* 临时适配旧的对话记录,清洗完数据后可删除（4.30刪除） */
+/* 临时适配旧的对话记录 */
 export const adaptStringValue = (value: any): ChatItemValueItemType[] => {
   if (typeof value === 'string') {
     return [

packages/service/core/chat/inputGuide/schema.ts

@@ -0,0 +1,29 @@
+import { AppCollectionName } from '../../app/schema';
+import { connectionMongo, type Model } from '../../../common/mongo';
+const { Schema, model, models } = connectionMongo;
+import type { ChatInputGuideSchemaType } from '@fastgpt/global/core/chat/inputGuide/type.d';
+
+export const ChatInputGuideCollectionName = 'chat_input_guides';
+
+const ChatInputGuideSchema = new Schema({
+  appId: {
+    type: Schema.Types.ObjectId,
+    ref: AppCollectionName,
+    required: true
+  },
+  text: {
+    type: String,
+    default: ''
+  }
+});
+
+try {
+  ChatInputGuideSchema.index({ appId: 1, text: 1 }, { unique: true });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoChatInputGuide: Model<ChatInputGuideSchemaType> =
+  models[ChatInputGuideCollectionName] || model(ChatInputGuideCollectionName, ChatInputGuideSchema);
+
+MongoChatInputGuide.syncIndexes();

packages/service/core/dataset/collection/controller.ts

@@ -32,6 +32,9 @@ export async function createOneCollection({
   fileId,
   rawLink,
 
+  externalFileId,
+  externalFileUrl,
+
   hashRawText,
   rawTextLength,
   metadata = {},
@@ -61,6 +64,8 @@ export async function createOneCollection({
 
         fileId,
         rawLink,
+        externalFileId,
+        externalFileUrl,
 
         rawTextLength,
         hashRawText,

packages/service/core/dataset/collection/schema.ts

@@ -8,7 +8,7 @@ import {
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-export const DatasetColCollectionName = 'dataset.collections';
+export const DatasetColCollectionName = 'dataset_collections';
 
 const DatasetCollectionSchema = new Schema({
   parentId: {
@@ -16,11 +16,6 @@ const DatasetCollectionSchema = new Schema({
     ref: DatasetColCollectionName,
     default: null
   },
-  userId: {
-    // abandoned
-    type: Schema.Types.ObjectId,
-    ref: 'user'
-  },
   teamId: {
     type: Schema.Types.ObjectId,
     ref: TeamCollectionName,
@@ -54,6 +49,7 @@ const DatasetCollectionSchema = new Schema({
     default: () => new Date()
   },
 
+  // chunk filed
   trainingType: {
     type: String,
     enum: Object.keys(TrainingTypeMap),
@@ -70,20 +66,25 @@ const DatasetCollectionSchema = new Schema({
     type: String
   },
 
+  tags: {
+    type: [String],
+    default: []
+  },
+
+  // local file collection
   fileId: {
     type: Schema.Types.ObjectId,
     ref: 'dataset.files'
   },
-  rawLink: {
+  // web link collection
-    type: String
+  rawLink: String,
-  },
+  // external collection
+  externalFileId: String,
 
-  rawTextLength: {
+  // metadata
-    type: Number
+  rawTextLength: Number,
-  },
+  hashRawText: String,
-  hashRawText: {
+  externalFileUrl: String, // external import url
-    type: String
-  },
   metadata: {
     type: Object,
     default: {}

packages/service/core/dataset/data/schema.ts

@@ -8,7 +8,7 @@ import {
 import { DatasetCollectionName } from '../schema';
 import { DatasetColCollectionName } from '../collection/schema';
 
-export const DatasetDataCollectionName = 'dataset.datas';
+export const DatasetDataCollectionName = 'dataset_datas';
 
 const DatasetDataSchema = new Schema({
   teamId: {
@@ -73,7 +73,8 @@ const DatasetDataSchema = new Schema({
   },
   inited: {
     type: Boolean
-  }
+  },
+  rebuilding: Boolean
 });
 
 try {
@@ -90,10 +91,13 @@ try {
     { background: true }
   );
   DatasetDataSchema.index({ updateTime: 1 }, { background: true });
+  // rebuild data
+  DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }, { background: true });
 } catch (error) {
   console.log(error);
 }
 
 export const MongoDatasetData: Model<DatasetDataSchemaType> =
   models[DatasetDataCollectionName] || model(DatasetDataCollectionName, DatasetDataSchema);
+
 MongoDatasetData.syncIndexes();

packages/service/core/dataset/read.ts

@@ -0,0 +1,112 @@
+import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { readFileContentFromMongo } from '../../common/file/gridfs/controller';
+import { urlsFetch } from '../../common/string/cheerio';
+import { parseCsvTable2Chunks } from './training/utils';
+import { TextSplitProps, splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
+import axios from 'axios';
+import { readRawContentByFileBuffer } from '../../common/file/read/utils';
+
+export const readFileRawTextByUrl = async ({
+  teamId,
+  url,
+  relatedId
+}: {
+  teamId: string;
+  url: string;
+  relatedId?: string;
+}) => {
+  const response = await axios({
+    method: 'get',
+    url: url,
+    responseType: 'arraybuffer'
+  });
+  const extension = url.split('.')?.pop()?.toLowerCase() || '';
+
+  const buffer = Buffer.from(response.data, 'binary');
+
+  const { rawText } = await readRawContentByFileBuffer({
+    extension,
+    teamId,
+    buffer,
+    encoding: 'utf-8',
+    metadata: {
+      relatedId
+    }
+  });
+
+  return rawText;
+};
+
+/* 
+    fileId - local file, read from mongo
+    link - request
+    externalFile = request read
+*/
+export const readDatasetSourceRawText = async ({
+  teamId,
+  type,
+  sourceId,
+  isQAImport,
+  selector,
+  relatedId
+}: {
+  teamId: string;
+  type: DatasetSourceReadTypeEnum;
+  sourceId: string;
+  isQAImport?: boolean;
+  selector?: string;
+  relatedId?: string;
+}): Promise<string> => {
+  if (type === DatasetSourceReadTypeEnum.fileLocal) {
+    const { rawText } = await readFileContentFromMongo({
+      teamId,
+      bucketName: BucketNameEnum.dataset,
+      fileId: sourceId,
+      isQAImport
+    });
+    return rawText;
+  } else if (type === DatasetSourceReadTypeEnum.link) {
+    const result = await urlsFetch({
+      urlList: [sourceId],
+      selector
+    });
+
+    return result[0]?.content || '';
+  } else if (type === DatasetSourceReadTypeEnum.externalFile) {
+    const rawText = await readFileRawTextByUrl({
+      teamId,
+      url: sourceId,
+      relatedId
+    });
+    return rawText;
+  }
+
+  return '';
+};
+
+export const rawText2Chunks = ({
+  rawText,
+  isQAImport,
+  chunkLen = 512,
+  ...splitProps
+}: {
+  rawText: string;
+  isQAImport?: boolean;
+} & TextSplitProps) => {
+  if (isQAImport) {
+    const { chunks } = parseCsvTable2Chunks(rawText);
+    return chunks;
+  }
+
+  const { chunks } = splitText2Chunks({
+    text: rawText,
+    chunkLen,
+    ...splitProps
+  });
+
+  return chunks.map((item) => ({
+    q: item,
+    a: ''
+  }));
+};

packages/service/core/dataset/schema.ts

@@ -89,7 +89,8 @@ const DatasetSchema = new Schema({
         default: 'body'
       }
     }
-  }
+  },
+  externalReadUrl: String
 });
 
 try {

packages/service/core/dataset/search/controller.ts

@@ -18,6 +18,7 @@ import { countPromptTokens } from '../../../common/string/tiktoken/index';
 import { datasetSearchResultConcat } from '@fastgpt/global/core/dataset/search/utils';
 import { hashStr } from '@fastgpt/global/common/string/tools';
 import { jiebaSplit } from '../../../common/string/jieba';
+import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
 
 type SearchDatasetDataProps = {
   teamId: string;
@@ -98,7 +99,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
       },
       'datasetId collectionId q a chunkIndex indexes'
     )
-      .populate('collectionId', 'name fileId rawLink')
+      .populate('collectionId', 'name fileId rawLink externalFileId externalFileUrl')
       .lean()) as DatasetDataWithCollectionType[];
 
     // add score to data(It's already sorted. The first one is the one with the most points)
@@ -130,8 +131,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
           chunkIndex: data.chunkIndex,
           datasetId: String(data.datasetId),
           collectionId: String(data.collectionId?._id),
-          sourceName: data.collectionId?.name || '',
+          ...getCollectionSourceData(data.collectionId),
-          sourceId: data.collectionId?.fileId || data.collectionId?.rawLink,
           score: [{ type: SearchScoreTypeEnum.embedding, value: data.score, index }]
         };
 
@@ -205,8 +205,7 @@ export async function searchDatasetData(props: SearchDatasetDataProps) {
           id: String(item._id),
           datasetId: String(item.datasetId),
           collectionId: String(item.collectionId),
-          sourceName: collection?.name || '',
+          ...getCollectionSourceData(collection),
-          sourceId: collection?.fileId || collection?.rawLink,
           q: item.q,
           a: item.a,
           chunkIndex: item.chunkIndex,

packages/service/core/dataset/search/utils.ts

@@ -2,6 +2,7 @@ import { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
 import { queryExtension } from '../../ai/functions/queryExtension';
 import { ChatItemType } from '@fastgpt/global/core/chat/type';
 import { hashStr } from '@fastgpt/global/common/string/tools';
+import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
 
 export const datasetSearchQueryExtension = async ({
   query,
@@ -33,11 +34,11 @@ export const datasetSearchQueryExtension = async ({
       histories.length > 0
         ? `${histories
             .map((item) => {
-              return `${item.obj}: ${item.value}`;
+              return `${item.obj}: ${chatValue2RuntimePrompt(item.value).text}`;
             })
             .join('\n')}
-  Human: ${query}
+Human: ${query}
-  `
+`
         : query;
 
     /* if query already extension, direct parse */

packages/service/core/dataset/training/controller.ts

@@ -168,13 +168,14 @@ export async function pushDataListToTrainingQueue({
         indexes: item.indexes
       })),
       {
-        session
+        session,
+        ordered: false
       }
     );
   } catch (error: any) {
     addLog.error(`Insert error`, error);
     // 如果有错误，将失败的文档添加到失败列表中
-    error.writeErrors.forEach((writeError: any) => {
+    error.writeErrors?.forEach((writeError: any) => {
       failedDocuments.push(data[writeError.index]);
     });
     console.log('failed', failedDocuments);

packages/service/core/dataset/training/schema.ts

@@ -10,7 +10,7 @@ import {
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-export const DatasetTrainingCollectionName = 'dataset.trainings';
+export const DatasetTrainingCollectionName = 'dataset_trainings';
 
 const TrainingDataSchema = new Schema({
   teamId: {
@@ -35,8 +35,7 @@ const TrainingDataSchema = new Schema({
   },
   billId: {
     // concat bill
-    type: String,
+    type: String
-    default: ''
   },
   mode: {
     type: String,
@@ -78,6 +77,9 @@ const TrainingDataSchema = new Schema({
     type: Number,
     default: 0
   },
+  dataId: {
+    type: Schema.Types.ObjectId
+  },
   indexes: {
     type: [
       {

packages/service/core/plugin/controller.ts

@@ -52,7 +52,8 @@ const getPluginTemplateById = async (id: string): Promise<PluginTemplateType> =>
       nodes: item.modules,
       edges: item.edges,
       templateType: FlowNodeTemplateTypeEnum.personalPlugin,
-      isTool: true
+      isTool: true,
+      nodeVersion: item?.nodeVersion || ''
     };
   }
   return Promise.reject('plugin not found');
@@ -72,6 +73,8 @@ export async function getPluginPreviewNode({ id }: { id: string }): Promise<Flow
     intro: plugin.intro,
     showStatus: plugin.showStatus,
     isTool: plugin.isTool,
+    nodeVersion: plugin.nodeVersion,
+    version: '481',
     sourceHandle: getHandleConfig(true, true, true, true),
     targetHandle: getHandleConfig(true, true, true, true),
     ...pluginData2FlowNodeIO(plugin.nodes)

packages/service/core/plugin/schema.ts

@@ -64,6 +64,10 @@ const PluginSchema = new Schema({
   version: {
     type: String,
     enum: ['v1', 'v2']
+  },
+  nodeVersion: {
+    type: String,
+    default: ''
   }
 });
 

packages/service/core/workflow/dispatch/dataset/search.ts

@@ -15,6 +15,7 @@ import { getHistories } from '../utils';
 import { datasetSearchQueryExtension } from '../../../dataset/search/utils';
 import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
 import { checkTeamReRankPermission } from '../../../../support/permission/teamLimit';
+import { MongoDataset } from '../../../dataset/schema';
 
 type DatasetSearchProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
@@ -79,7 +80,9 @@ export async function dispatchDatasetSearch(
   // console.log(concatQueries, rewriteQuery, aiExtensionResult);
 
   // get vector
-  const vectorModel = getVectorModel(datasets[0]?.vectorModel?.model);
+  const vectorModel = getVectorModel(
+    (await MongoDataset.findById(datasets[0].datasetId, 'vectorModel').lean())?.vectorModel
+  );
 
   // start search
   const {

packages/service/core/workflow/dispatch/index.ts

@@ -44,8 +44,9 @@ import { RuntimeEdgeItemType } from '@fastgpt/global/core/workflow/type/edge';
 import { getReferenceVariableValue } from '@fastgpt/global/core/workflow/runtime/utils';
 import { dispatchSystemConfig } from './init/systemConfig';
 import { dispatchUpdateVariable } from './tools/runUpdateVar';
+import { addLog } from '../../../common/system/log';
 
-const callbackMap: Record<`${FlowNodeTypeEnum}`, Function> = {
+const callbackMap: Record<FlowNodeTypeEnum, Function> = {
   [FlowNodeTypeEnum.workflowStart]: dispatchWorkflowStart,
   [FlowNodeTypeEnum.answerNode]: dispatchAnswer,
   [FlowNodeTypeEnum.chatNode]: dispatchChatCompletion,
@@ -137,7 +138,6 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
     }
     if (nodeDispatchUsages) {
       chatNodeUsages = chatNodeUsages.concat(nodeDispatchUsages);
-      props.maxRunTimes -= nodeDispatchUsages.length;
     }
     if (toolResponses !== undefined) {
       if (Array.isArray(toolResponses) && toolResponses.length === 0) return;
@@ -213,9 +213,11 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
         });
 
         if (status === 'run') {
+          addLog.info(`[dispatchWorkFlow] nodeRunWithActive: ${node.name}`);
           return nodeRunWithActive(node);
         }
         if (status === 'skip') {
+          addLog.info(`[dispatchWorkFlow] nodeRunWithActive: ${node.name}`);
           return nodeRunWithSkip(node);
         }
 
@@ -275,6 +277,8 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
       });
     }
 
+    props.maxRunTimes--;
+
     // get node running params
     const params = getNodeRunParams(node);
 

packages/service/core/workflow/dispatch/tools/http468.ts

@@ -71,7 +71,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
     chatId,
     responseChatItemId,
     ...variables,
-    histories: histories.slice(-10),
+    histories: histories?.slice(-10) || [],
     ...body,
     ...dynamicInput
   };

packages/service/core/workflow/dispatch/tools/runLaf.ts

@@ -53,7 +53,7 @@ export const dispatchLafRequest = async (props: LafRequestProps): Promise<LafRes
       appId,
       chatId,
       responseChatItemId,
-      histories: histories.slice(0, 10)
+      histories: histories?.slice(0, 10)
     },
     variables,
     ...dynamicInput,

packages/service/core/workflow/dispatch/utils.ts

@@ -62,7 +62,10 @@ export const valueTypeFormat = (value: any, type?: WorkflowIOValueTypeEnum) => {
     return JSON.stringify(value);
   }
   if (type === 'number') return Number(value);
-  if (type === 'boolean') return Boolean(value);
+  if (type === 'boolean') {
+    if (typeof value === 'string') return value === 'true';
+    return Boolean(value);
+  }
   try {
     if (type === WorkflowIOValueTypeEnum.datasetQuote && !Array.isArray(value)) {
       return JSON.parse(value);