4.8 test fix (#1397)

* adapt v1 chat init

* adapt v1 chat init

* adapt v1 chat init

* perf: message input line; fix: http request un stream

* perf: message input line; fix: http request un stream

* perf: message input line; fix: http request un stream

* perf: error tip

.github/ISSUE_TEMPLATE/bugs.md

@@ -21,7 +21,7 @@ assignees: ''
 - [ ] 公有云版本
 - [ ] 私有部署版本, 具体版本号: 
 
-**问题描述**
+**问题描述, 日志截图**
 
 **复现步骤**
 

.vscode/nextapi.code-snippets

@@ -0,0 +1,29 @@
+{
+	// Place your FastGPT 工作区 snippets here. Each snippet is defined under a snippet name and has a scope, prefix, body and 
+	// description. Add comma separated ids of the languages where the snippet is applicable in the scope field. If scope 
+	// is left empty or omitted, the snippet gets applied to all languages. The prefix is what is 
+	// used to trigger the snippet and the body will be expanded and inserted. Possible variables are: 
+	// $1, $2 for tab stops, $0 for the final cursor position, and ${1:label}, ${2:another} for placeholders. 
+	// Placeholders with the same ids are connected.
+	// Example:
+	"Next api template": {
+		"scope": "javascript,typescript",
+		"prefix": "nextapi",
+		"body": [
+			"import type { NextApiRequest, NextApiResponse } from 'next';",
+			"import { NextAPI } from '@/service/middle/entry';",
+			"",
+			"type Props = {};",
+			"",
+			"type Response = {};",
+			"",
+			"async function handler(req: NextApiRequest, res: NextApiResponse<any>): Promise<Response> {",
+			"    $1",
+			"    return {}",
+			"}",
+			"",
+			"export default NextAPI(handler);"
+		],
+		"description": "FastGPT Next API template"
+	}
+}

.vscode/settings.json

@@ -4,12 +4,12 @@
     "typescript.tsdk": "node_modules/typescript/lib",
     "prettier.prettierPath": "",
     "i18n-ally.localesPaths": [
-        "projects/app/public/locales",
+        "projects/app/i18n",
     ],
-    "i18n-ally.enabledParsers": ["json"],
+    "i18n-ally.enabledParsers": ["json", "yaml", "js", "ts"],
     "i18n-ally.keystyle": "nested",
     "i18n-ally.sortKeys": true,
-    "i18n-ally.keepFulfilled": true,
+    "i18n-ally.keepFulfilled": false,
     "i18n-ally.sourceLanguage": "zh", // 根据此语言文件翻译其他语言文件的变量和内容
-    "i18n-ally.displayLanguage": "zh", // 显示语言
+    "i18n-ally.displayLanguage": "zh" // 显示语言
 }

docSite/content/docs/development/faq.md

@@ -13,7 +13,8 @@ images: []
 
 1. `docker ps -a` 查看所有容器运行状态，检查是否全部 running，如有异常，尝试`docker logs 容器名`查看对应日志。
 2. 容器都运行正常的，`docker logs 容器名` 查看报错日志
-3. 无法解决时，可以找找[Issue](https://github.com/labring/FastGPT/issues)，或新提 Issue，私有部署错误，务必提供详细的日志，否则很难排查。
+3. 带有`requestId`的，都是 OneAPI 提示错误，大部分都是因为模型接口报错。
+4. 无法解决时，可以找找[Issue](https://github.com/labring/FastGPT/issues)，或新提 Issue，私有部署错误，务必提供详细的日志，否则很难排查。
 
 
 ## 二、通用问题
@@ -44,7 +45,7 @@ images: []
 
 ### 模型响应为空(core.chat.Chat API is error or undefined)
 
-1. 检查 key 问题。
+1. 检查 key 问题。curl 请求看是否正常。务必用 stream=true 模式。并且 maxToken 等相关参数尽量一致。
 2. 如果是国内模型，可能是命中风控了。
 3. 查看模型请求日志，检查出入参数是否异常。
 
@@ -90,4 +91,9 @@ FastGPT 模型配置文件中的 model 必须与 OneAPI 渠道中的模型对应
 
 OneAPI 的 API Key 配置错误，需要修改`OPENAI_API_KEY`环境变量，并重启容器（先 docker-compose down 然后再 docker-compose up -d 运行一次）。
 
-可以`exec`进入容器，`env`查看环境变量是否生效。
+可以`exec`进入容器，`env`查看环境变量是否生效。
+
+### bad_response_status_code bad response status code 503
+
+1. 模型服务不可用
+2. ....

docSite/content/docs/development/sealos.md

@@ -106,6 +106,7 @@ FastGPT 商业版共包含了2个应用（fastgpt, fastgpt-plus）和2个数据
 
 ```
 SYSTEM_NAME=FastGPT
+SYSTEM_DESCRIPTION=
 SYSTEM_FAVICON=/favicon.ico
 HOME_URL=/app/list
 ```

docSite/content/docs/development/upgrading/48.md

@@ -1,5 +1,5 @@
 ---
-title: 'V4.8(进行中)'
+title: 'V4.8(开发中)'
 description: 'FastGPT V4.8 更新说明'
 icon: 'upgrade'
 draft: false
@@ -18,8 +18,14 @@ FastGPT workflow V2上线，支持更加简洁的工作流模式。
 ## V4.8 更新说明
 
 1. 重构 - 工作流
-2. 新增 - 工作流 Debug 模式，可以调试单个节点或者逐步调试工作流。
+2. 新增 - 判断器。支持 if elseIf else 判断。
-3. 新增 - 定时执行应用。可轻松实现定时任务。
+3. 新增 - 变量更新节点。支持更新运行中工作流输出变量，或更新全局变量。
-4. 新增 - 插件自定义输入优化，可以渲染输入组件。
+4. 新增 - 工作流 Debug 模式，可以调试单个节点或者逐步调试工作流。
-5. 优化 - 工作流连线，可以四向连接，方便构建循环工作流。
+5. 新增 - 定时执行应用。可轻松实现定时任务。
-6. 优化 - worker进程管理，并将计算 Token 任务分配给 worker 进程。
+6. 新增 - 插件自定义输入优化，可以渲染输入组件。
+7. 优化 - 工作流连线，可以四向连接，方便构建循环工作流。
+8. 优化 - 工作流上下文传递，性能🚀。
+9. 优化 - 简易模式，更新配置后自动更新调试框内容，无需保存。
+10. 优化 - worker进程管理，并将计算 Token 任务分配给 worker 进程。
+11. 修复 - 工具调用时候，name不能是数字开头（随机数有概率数字开头）
+12. 修复 - 分享链接， query 全局变量会被缓存。

packages/global/common/error/code/app.ts

@@ -2,7 +2,7 @@ import { ErrType } from '../errorCode';
 
 /* dataset: 502000 */
 export enum AppErrEnum {
-  unExist = 'unExist',
+  unExist = 'appUnExist',
   unAuthApp = 'unAuthApp'
 }
 const appErrList = [

packages/global/common/error/code/openapi.ts

@@ -2,8 +2,8 @@ import { ErrType } from '../errorCode';
 
 /* dataset: 506000 */
 export enum OpenApiErrEnum {
-  unExist = 'unExist',
+  unExist = 'openapiUnExist',
-  unAuth = 'unAuth'
+  unAuth = 'openapiUnAuth'
 }
 const errList = [
   {

packages/global/common/error/code/outLink.ts

@@ -2,7 +2,7 @@ import { ErrType } from '../errorCode';
 
 /* dataset: 505000 */
 export enum OutLinkErrEnum {
-  unExist = 'unExist',
+  unExist = 'outlinkUnExist',
   unAuthLink = 'unAuthLink',
   linkUnInvalid = 'linkUnInvalid',
 

packages/global/common/error/code/plugin.ts

@@ -2,8 +2,8 @@ import { ErrType } from '../errorCode';
 
 /* dataset: 507000 */
 export enum PluginErrEnum {
-  unExist = 'unExist',
+  unExist = 'pluginUnExist',
-  unAuth = 'unAuth'
+  unAuth = 'pluginUnAuth'
 }
 const errList = [
   {

packages/global/common/string/time.ts

@@ -4,6 +4,7 @@ import cronParser from 'cron-parser';
 export const formatTime2YMDHM = (time?: Date) =>
   time ? dayjs(time).format('YYYY-MM-DD HH:mm') : '';
 export const formatTime2YMD = (time?: Date) => (time ? dayjs(time).format('YYYY-MM-DD') : '');
+export const formatTime2HM = (time: Date = new Date()) => dayjs(time).format('HH:mm');
 
 /* cron time parse */
 export const cronParser2Fields = (cronString: string) => {

packages/global/common/string/tools.ts

@@ -50,8 +50,18 @@ export const replaceSensitiveText = (text: string) => {
   return text;
 };
 
+/* Make sure the first letter is definitely lowercase */
 export const getNanoid = (size = 12) => {
-  return customAlphabet('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', size)();
+  const firstChar = customAlphabet('abcdefghijklmnopqrstuvwxyz', 1)();
+
+  if (size === 1) return firstChar;
+
+  const randomsStr = customAlphabet(
+    'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890',
+    size - 1
+  )();
+
+  return `${firstChar}${randomsStr}`;
 };
 
 export const replaceRegChars = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');

packages/global/common/system/types/index.d.ts

@@ -37,6 +37,7 @@ export type FastGPTFeConfigsType = {
   chatbotUrl?: string;
   openAPIDocUrl?: string;
   systemTitle?: string;
+  systemDescription?: string;
   googleClientVerKey?: string;
   isPlus?: boolean;
   show_phoneLogin?: boolean;

packages/global/core/app/api.d.ts

@@ -1,22 +0,0 @@
-import type { LLMModelItemType } from '../ai/model.d';
-import { AppTypeEnum } from './constants';
-import { AppSchema } from './type';
-
-export type CreateAppParams = {
-  name?: string;
-  avatar?: string;
-  type?: `${AppTypeEnum}`;
-  modules: AppSchema['modules'];
-  edges?: AppSchema['edges'];
-};
-
-export interface AppUpdateParams {
-  name?: string;
-  type?: `${AppTypeEnum}`;
-  avatar?: string;
-  intro?: string;
-  modules?: AppSchema['modules'];
-  edges?: AppSchema['edges'];
-  permission?: AppSchema['permission'];
-  teamTags?: AppSchema['teamTags'];
-}

packages/global/core/app/type.d.ts

@@ -6,7 +6,7 @@ import { VariableInputEnum } from '../workflow/constants';
 import { SelectedDatasetType } from '../workflow/api';
 import { DatasetSearchModeEnum } from '../dataset/constants';
 import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
-import { StoreEdgeItemType } from 'core/workflow/type/edge';
+import { StoreEdgeItemType } from '../workflow/type/edge';
 
 export interface AppSchema {
   _id: string;
@@ -18,6 +18,7 @@ export interface AppSchema {
   avatar: string;
   intro: string;
   updateTime: number;
+
   modules: StoreNodeItemType[];
   edges: StoreEdgeItemType[];
 

packages/global/core/app/utils.ts

@@ -105,7 +105,7 @@ export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => {
     } else if (node.flowNodeType === FlowNodeTypeEnum.systemConfig) {
       const {
         welcomeText,
-        variableModules,
+        variableNodes,
         questionGuide,
         ttsConfig,
         whisperConfig,
@@ -114,7 +114,7 @@ export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => {
 
       defaultAppForm.userGuide = {
         welcomeText: welcomeText,
-        variables: variableModules,
+        variables: variableNodes,
         questionGuide: questionGuide,
         tts: ttsConfig,
         whisper: whisperConfig,
@@ -125,6 +125,7 @@ export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => {
 
       defaultAppForm.selectedTools.push({
         id: node.pluginId,
+        pluginId: node.pluginId,
         name: node.name,
         avatar: node.avatar,
         intro: node.intro || '',

packages/global/core/app/version.d.ts

@@ -0,0 +1,10 @@
+import { StoreNodeItemType } from '../workflow/type';
+import { StoreEdgeItemType } from '../workflow/type/edge';
+
+export type AppVersionSchemaType = {
+  _id: string;
+  appId: string;
+  time: Date;
+  nodes: StoreNodeItemType[];
+  edges: StoreEdgeItemType[];
+};

packages/global/core/chat/type.d.ts

@@ -10,7 +10,7 @@ import {
 import { FlowNodeTypeEnum } from '../workflow/node/constant';
 import { NodeOutputKeyEnum } from '../workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '../workflow/runtime/constants';
-import { AppSchema } from '../app/type';
+import { AppSchema, VariableItemType } from '../app/type';
 import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
 import { DatasetSearchModeEnum } from '../dataset/constants';
 import { ChatBoxInputType } from '../../../../projects/app/src/components/ChatBox/type';
@@ -27,11 +27,13 @@ export type ChatSchema = {
   title: string;
   customTitle: string;
   top: boolean;
-  variables: Record<string, any>;
   source: `${ChatSourceEnum}`;
   shareId?: string;
   outLinkUid?: string;
-  content: ChatItemType[];
+
+  variableList?: VariableItemType[];
+  welcomeText?: string;
+  variables: Record<string, any>;
   metadata?: Record<string, any>;
 };
 
@@ -155,6 +157,6 @@ export type ToolModuleResponseItemType = {
 
 /* dispatch run time */
 export type RuntimeUserPromptType = {
-  files?: UserChatItemValueItemType['file'][];
+  files: UserChatItemValueItemType['file'][];
   text: string;
 };

packages/global/core/chat/utils.ts

@@ -1,7 +1,7 @@
 import { DispatchNodeResponseType } from '../workflow/runtime/type';
-import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../workflow/node/constant';
+import { FlowNodeTypeEnum } from '../workflow/node/constant';
 import { ChatItemValueTypeEnum, ChatRoleEnum } from './constants';
-import { ChatHistoryItemResType, ChatItemType } from './type.d';
+import { ChatHistoryItemResType, ChatItemType, UserChatItemValueItemType } from './type.d';
 
 export const getChatTitleFromChatMessage = (message?: ChatItemType, defaultValue = '新对话') => {
   // @ts-ignore
@@ -77,3 +77,15 @@ export const filterPublicNodeResponseData = ({
       return obj as ChatHistoryItemResType;
     });
 };
+
+export const removeEmptyUserInput = (input: UserChatItemValueItemType[]) => {
+  return input.filter((item) => {
+    if (item.type === ChatItemValueTypeEnum.text && !item.text?.content?.trim()) {
+      return false;
+    }
+    if (item.type === ChatItemValueTypeEnum.file && !item.file?.url) {
+      return false;
+    }
+    return true;
+  });
+};

packages/global/core/plugin/httpPlugin/utils.ts

@@ -14,6 +14,7 @@ import { CreateOnePluginParams } from '../controller';
 import { StoreNodeItemType } from '../../workflow/type';
 import { HttpImgUrl } from '../../../common/file/image/constants';
 import SwaggerParser from '@apidevtools/swagger-parser';
+import { getHandleId } from '../../../core/workflow/utils';
 
 export const str2OpenApiSchema = async (yamlStr = ''): Promise<OpenApiJsonSchema> => {
   try {
@@ -378,14 +379,14 @@ export const httpApiSchema2Plugins = async ({
       {
         source: pluginInputId,
         target: httpId,
-        sourcePort: `${pluginInputId}-source-right`,
+        sourceHandle: getHandleId(pluginInputId, 'source', 'right'),
-        targetPort: `${httpId}-target-left`
+        targetHandle: getHandleId(httpId, 'target', 'left')
       },
       {
         source: httpId,
         target: pluginOutputId,
-        sourcePort: `${httpId}-source-right`,
+        sourceHandle: getHandleId(httpId, 'source', 'right'),
-        targetPort: `${pluginOutputId}-target-left`
+        targetHandle: getHandleId(pluginOutputId, 'target', 'left')
       }
     ];
 

packages/global/core/workflow/constants.ts

@@ -14,18 +14,21 @@ export enum WorkflowIOValueTypeEnum {
   string = 'string',
   number = 'number',
   boolean = 'boolean',
+  object = 'object',
+  arrayString = 'arrayString',
+  arrayNumber = 'arrayNumber',
+  arrayBoolean = 'arrayBoolean',
+  arrayObject = 'arrayObject',
   any = 'any',
 
   chatHistory = 'chatHistory',
   datasetQuote = 'datasetQuote',
+
   dynamic = 'dynamic',
 
   // plugin special type
   selectApp = 'selectApp',
-  selectDataset = 'selectDataset',
+  selectDataset = 'selectDataset'
-
-  // tool
-  tools = 'tools'
 }
 
 /* reg: modulename key */
@@ -34,7 +37,6 @@ export enum NodeInputKeyEnum {
   welcomeText = 'welcomeText',
   switch = 'switch', // a trigger switch
   history = 'history',
-  userChatInput = 'userChatInput',
   answerText = 'text',
 
   // system config
@@ -44,6 +46,10 @@ export enum NodeInputKeyEnum {
   variables = 'variables',
   scheduleTrigger = 'scheduleTrigger',
 
+  // entry
+  userChatInput = 'userChatInput',
+  inputFiles = 'inputFiles',
+
   agents = 'agents', // cq agent key
 
   // latest
@@ -98,7 +104,10 @@ export enum NodeInputKeyEnum {
 
   // if else
   condition = 'condition',
-  ifElseList = 'ifElseList'
+  ifElseList = 'ifElseList',
+
+  // variable update
+  updateList = 'updateList'
 }
 
 export enum NodeOutputKeyEnum {
@@ -132,15 +141,14 @@ export enum NodeOutputKeyEnum {
   // plugin
   pluginStart = 'pluginStart',
 
-  if = 'IF',
+  ifElseResult = 'ifElseResult'
-  else = 'ELSE'
 }
 
 export enum VariableInputEnum {
   input = 'input',
   textarea = 'textarea',
   select = 'select',
-  external = 'external'
+  custom = 'custom'
 }
 export const variableMap = {
   [VariableInputEnum.input]: {
@@ -158,10 +166,10 @@ export const variableMap = {
     title: 'core.module.variable.select type',
     desc: ''
   },
-  [VariableInputEnum.external]: {
+  [VariableInputEnum.custom]: {
     icon: 'core/app/variable/external',
-    title: 'core.module.variable.External type',
+    title: 'core.module.variable.Custom type',
-    desc: '可以通过API接口或分享链接的Query传递变量。增加该类型变量的主要目的是用于变量提示。使用例子: 你可以通过分享链接Query中拼接Token，来实现内部系统身份鉴权。'
+    desc: '可以定义一个无需用户填写的全局变量。\n该变量的值可以来自于 API 接口，分享链接的 Query 或通过【变量更新】模块进行赋值。'
   }
 };
 
@@ -173,3 +181,5 @@ export enum RuntimeEdgeStatusEnum {
   'active' = 'active',
   'skipped' = 'skipped'
 }
+
+export const VARIABLE_NODE_ID = 'VARIABLE_NODE_ID';

packages/global/core/workflow/node/constant.ts

@@ -112,7 +112,8 @@ export enum FlowNodeTypeEnum {
   tools = 'tools',
   stopTool = 'stopTool',
   lafModule = 'lafModule',
-  ifElseNode = 'ifElseNode'
+  ifElseNode = 'ifElseNode',
+  variableUpdate = 'variableUpdate'
 }
 
 export const EDGE_TYPE = 'default';

packages/global/core/workflow/runtime/constants.ts

@@ -9,7 +9,8 @@ export enum SseResponseEventEnum {
   toolCall = 'toolCall', // tool start
   toolParams = 'toolParams', // tool params return
   toolResponse = 'toolResponse', // tool response return
-  flowResponses = 'flowResponses' // sse response request
+  flowResponses = 'flowResponses', // sse response request
+  updateVariables = 'updateVariables'
 }
 
 export enum DispatchNodeResponseKeyEnum {

packages/global/core/workflow/runtime/type.d.ts

@@ -75,7 +75,7 @@ export type DispatchNodeResponseType = {
   pluginDetail?: ChatHistoryItemResType[];
 
   // if-else
-  ifElseResult?: 'IF' | 'ELSE';
+  ifElseResult?: string;
 
   // tool
   toolCallTokens?: number;

packages/global/core/workflow/runtime/utils.ts

@@ -4,7 +4,9 @@ import { FlowNodeTypeEnum } from '../node/constant';
 import { StoreNodeItemType } from '../type';
 import { StoreEdgeItemType } from '../type/edge';
 import { RuntimeEdgeItemType, RuntimeNodeItemType } from './type';
-import { VARIABLE_NODE_ID } from '../../../../../projects/app/src/web/core/workflow/constants/index';
+import { VARIABLE_NODE_ID } from '../constants';
+import { isReferenceValue } from '../utils';
+import { ReferenceValueProps } from '../type/io';
 
 export const initWorkflowEdgeStatus = (edges: StoreEdgeItemType[]): RuntimeEdgeItemType[] => {
   return (
@@ -138,16 +140,11 @@ export const getReferenceVariableValue = ({
   nodes,
   variables
 }: {
-  value: [string, string];
+  value: ReferenceValueProps;
   nodes: RuntimeNodeItemType[];
   variables: Record<string, any>;
 }) => {
-  if (
+  if (!isReferenceValue(value)) {
-    !Array.isArray(value) ||
-    value.length !== 2 ||
-    typeof value[0] !== 'string' ||
-    typeof value[1] !== 'string'
-  ) {
     return value;
   }
   const sourceNodeId = value[0];

packages/global/core/workflow/template/constants.ts

@@ -18,10 +18,10 @@ import { PluginOutputModule } from './system/pluginOutput';
 import { RunPluginModule } from './system/runPlugin';
 import { AiQueryExtension } from './system/queryExtension';
 
-import type { FlowNodeTemplateType, nodeTemplateListType } from '../type';
+import type { FlowNodeTemplateType } from '../type';
-import { FlowNodeTemplateTypeEnum } from '../../workflow/constants';
+import { LafModule } from './system/laf';
-import { lafModule } from './system/laf';
+import { IfElseNode } from './system/ifElse/index';
-import { ifElseNode } from './system/ifElse/index';
+import { VariableUpdateNode } from './system/variableUpdate';
 
 /* app flow module templates */
 export const appSystemModuleTemplates: FlowNodeTemplateType[] = [
@@ -38,8 +38,9 @@ export const appSystemModuleTemplates: FlowNodeTemplateType[] = [
   ContextExtractModule,
   HttpModule468,
   AiQueryExtension,
-  lafModule,
+  LafModule,
-  ifElseNode
+  IfElseNode,
+  VariableUpdateNode
 ];
 /* plugin flow module templates */
 export const pluginSystemModuleTemplates: FlowNodeTemplateType[] = [
@@ -56,8 +57,9 @@ export const pluginSystemModuleTemplates: FlowNodeTemplateType[] = [
   ContextExtractModule,
   HttpModule468,
   AiQueryExtension,
-  lafModule,
+  LafModule,
-  ifElseNode
+  IfElseNode,
+  VariableUpdateNode
 ];
 
 /* all module */
@@ -80,44 +82,7 @@ export const moduleTemplatesFlat: FlowNodeTemplateType[] = [
   PluginOutputModule,
   RunPluginModule,
   AiQueryExtension,
-  lafModule,
+  LafModule,
-  ifElseNode
+  IfElseNode,
-];
+  VariableUpdateNode
-
-export const moduleTemplatesList: nodeTemplateListType = [
-  {
-    type: FlowNodeTemplateTypeEnum.systemInput,
-    label: 'core.module.template.System input module',
-    list: []
-  },
-  {
-    type: FlowNodeTemplateTypeEnum.textAnswer,
-    label: 'core.module.template.Response module',
-    list: []
-  },
-  {
-    type: FlowNodeTemplateTypeEnum.functionCall,
-    label: 'core.module.template.Function module',
-    list: []
-  },
-  {
-    type: FlowNodeTemplateTypeEnum.tools,
-    label: 'core.module.template.Tool module',
-    list: []
-  },
-  {
-    type: FlowNodeTemplateTypeEnum.externalCall,
-    label: 'core.module.template.External module',
-    list: []
-  },
-  {
-    type: FlowNodeTemplateTypeEnum.personalPlugin,
-    label: '',
-    list: []
-  },
-  {
-    type: FlowNodeTemplateTypeEnum.other,
-    label: '其他',
-    list: []
-  }
 ];

packages/global/core/workflow/template/input.ts

@@ -9,9 +9,10 @@ export const Input_Template_History: FlowNodeInputItemType = {
   renderTypeList: [FlowNodeInputTypeEnum.numberInput, FlowNodeInputTypeEnum.reference],
   valueType: WorkflowIOValueTypeEnum.chatHistory,
   label: 'core.module.input.label.chat history',
+  description: '最多携带多少轮对话记录',
   required: true,
   min: 0,
-  max: 30,
+  max: 50,
   value: 6
 };
 

packages/global/core/workflow/template/system/assignedAnswer.ts

@@ -1,14 +1,9 @@
-import {
+import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
-  FlowNodeInputTypeEnum,
-  FlowNodeOutputTypeEnum,
-  FlowNodeTypeEnum
-} from '../../node/constant';
 import { FlowNodeTemplateType } from '../../type/index.d';
 import {
   WorkflowIOValueTypeEnum,
   NodeInputKeyEnum,
-  FlowNodeTemplateTypeEnum,
+  FlowNodeTemplateTypeEnum
-  NodeOutputKeyEnum
 } from '../../constants';
 import { getHandleConfig } from '../utils';
 
@@ -26,7 +21,7 @@ export const AssignedAnswerModule: FlowNodeTemplateType = {
     {
       key: NodeInputKeyEnum.answerText,
       renderTypeList: [FlowNodeInputTypeEnum.textarea, FlowNodeInputTypeEnum.reference],
-      valueType: WorkflowIOValueTypeEnum.string,
+      valueType: WorkflowIOValueTypeEnum.any,
       label: 'core.module.input.label.Response content',
       description: 'core.module.input.description.Response content',
       placeholder: 'core.module.input.description.Response content'

packages/global/core/workflow/template/system/datasetConcat.ts

@@ -10,16 +10,26 @@ import {
   NodeOutputKeyEnum,
   FlowNodeTemplateTypeEnum
 } from '../../constants';
-import { Input_Template_Dataset_Quote } from '../input';
 import { getNanoid } from '../../../../common/string/tools';
 import { getHandleConfig } from '../utils';
 import { FlowNodeInputItemType } from '../../type/io.d';
 
-export const getOneQuoteInputTemplate = (key = getNanoid()): FlowNodeInputItemType => ({
+const defaultQuoteKey = 'defaultQuoteKey';
-  ...Input_Template_Dataset_Quote,
+
+export const getOneQuoteInputTemplate = ({
+  key = getNanoid(),
+  index
+}: {
+  key?: string;
+  index: number;
+}): FlowNodeInputItemType => ({
   key,
-  renderTypeList: [FlowNodeInputTypeEnum.custom],
+  renderTypeList: [FlowNodeInputTypeEnum.reference],
-  description: ''
+  label: `引用${index}`,
+  debugLabel: '知识库引用',
+  canEdit: key !== defaultQuoteKey,
+  description: '',
+  valueType: WorkflowIOValueTypeEnum.datasetQuote
 });
 
 export const DatasetConcatModule: FlowNodeTemplateType = {
@@ -37,7 +47,7 @@ export const DatasetConcatModule: FlowNodeTemplateType = {
       key: NodeInputKeyEnum.datasetMaxTokens,
       renderTypeList: [FlowNodeInputTypeEnum.custom],
       label: '最大 Tokens',
-      value: 1500,
+      value: 3000,
       valueType: WorkflowIOValueTypeEnum.number
     },
     {
@@ -45,7 +55,7 @@ export const DatasetConcatModule: FlowNodeTemplateType = {
       renderTypeList: [FlowNodeInputTypeEnum.custom],
       label: ''
     },
-    getOneQuoteInputTemplate()
+    getOneQuoteInputTemplate({ key: defaultQuoteKey, index: 1 })
   ],
   outputs: [
     {

packages/global/core/workflow/template/system/globalVariable.ts

@@ -1,6 +1,6 @@
 import { FlowNodeTemplateTypeEnum, WorkflowIOValueTypeEnum } from '../../constants';
 import { getHandleConfig } from '../utils';
-import { FlowNodeTypeEnum } from '../../node/constant';
+import { FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
 import { VariableItemType } from '../../../app/type';
 import { FlowNodeTemplateType } from '../../type';
 
@@ -25,6 +25,7 @@ export const getGlobalVariableNode = ({
       id: item.key,
       key: item.key,
       valueType: WorkflowIOValueTypeEnum.string,
+      type: FlowNodeOutputTypeEnum.static,
       label: item.label
     }))
   };

packages/global/core/workflow/template/system/ifElse/constant.ts

@@ -20,6 +20,11 @@ export enum VariableConditionEnum {
   lengthLessThan = 'lengthLessThan',
   lengthLessThanOrEqualTo = 'lengthLessThanOrEqualTo'
 }
+export enum IfElseResultEnum {
+  IF = 'IF',
+  ELSE = 'ELSE',
+  ELSE_IF = 'ELSE IF'
+}
 
 export const stringConditionList = [
   { label: '为空', value: VariableConditionEnum.isEmpty },

packages/global/core/workflow/template/system/ifElse/index.ts

@@ -12,7 +12,7 @@ import {
 import { FlowNodeTemplateType } from '../../../type';
 import { getHandleConfig } from '../../utils';
 
-export const ifElseNode: FlowNodeTemplateType = {
+export const IfElseNode: FlowNodeTemplateType = {
   id: FlowNodeTypeEnum.ifElseNode,
   templateType: FlowNodeTemplateTypeEnum.tools,
   flowNodeType: FlowNodeTypeEnum.ifElseNode,
@@ -23,14 +23,6 @@ export const ifElseNode: FlowNodeTemplateType = {
   intro: '根据一定的条件，执行不同的分支。',
   showStatus: true,
   inputs: [
-    {
-      key: NodeInputKeyEnum.condition,
-      valueType: WorkflowIOValueTypeEnum.string,
-      label: '',
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      required: false,
-      value: 'AND' // AND, OR
-    },
     {
       key: NodeInputKeyEnum.ifElseList,
       renderTypeList: [FlowNodeInputTypeEnum.hidden],
@@ -38,27 +30,25 @@ export const ifElseNode: FlowNodeTemplateType = {
       label: '',
       value: [
         {
-          variable: undefined,
+          condition: 'AND', // AND, OR
-          condition: undefined,
+          list: [
-          value: undefined
+            {
+              variable: undefined,
+              condition: undefined,
+              value: undefined
+            }
+          ]
         }
       ]
     }
   ],
   outputs: [
     {
-      id: NodeOutputKeyEnum.if,
+      id: NodeOutputKeyEnum.ifElseResult,
-      key: NodeOutputKeyEnum.if,
+      key: NodeOutputKeyEnum.ifElseResult,
-      label: 'IF',
+      label: '判断结果',
-      valueType: WorkflowIOValueTypeEnum.any,
+      valueType: WorkflowIOValueTypeEnum.string,
-      type: FlowNodeOutputTypeEnum.source
+      type: FlowNodeOutputTypeEnum.static
-    },
-    {
-      id: NodeOutputKeyEnum.else,
-      key: NodeOutputKeyEnum.else,
-      label: 'ELSE',
-      valueType: WorkflowIOValueTypeEnum.any,
-      type: FlowNodeOutputTypeEnum.source
     }
   ]
 };

packages/global/core/workflow/template/system/ifElse/type.d.ts

@@ -2,8 +2,12 @@ import { ReferenceValueProps } from 'core/workflow/type/io';
 import { VariableConditionEnum } from './constant';
 
 export type IfElseConditionType = 'AND' | 'OR';
-export type IfElseListItemType = {
+export type ConditionListItemType = {
   variable?: ReferenceValueProps;
   condition?: VariableConditionEnum;
   value?: string;
 };
+export type IfElseListItemType = {
+  condition: IfElseConditionType;
+  list: ConditionListItemType[];
+};

packages/global/core/workflow/template/system/laf.ts

@@ -14,7 +14,7 @@ import { Input_Template_DynamicInput } from '../input';
 import { Output_Template_AddOutput } from '../output';
 import { getHandleConfig } from '../utils';
 
-export const lafModule: FlowNodeTemplateType = {
+export const LafModule: FlowNodeTemplateType = {
   id: FlowNodeTypeEnum.lafModule,
   templateType: FlowNodeTemplateTypeEnum.externalCall,
   flowNodeType: FlowNodeTypeEnum.lafModule,

packages/global/core/workflow/template/system/pluginInput.ts

@@ -12,7 +12,7 @@ export const PluginInputModule: FlowNodeTemplateType = {
   unique: true,
   forbidDelete: true,
   avatar: '/imgs/workflow/input.png',
-  name: '定义插件输入',
+  name: '自定义插件输入',
   intro: '自定义配置外部输入，使用插件时，仅暴露自定义配置的输入',
   showStatus: false,
   inputs: [],

packages/global/core/workflow/template/system/pluginOutput.ts

@@ -12,7 +12,7 @@ export const PluginOutputModule: FlowNodeTemplateType = {
   unique: true,
   forbidDelete: true,
   avatar: '/imgs/workflow/output.png',
-  name: '定义插件输出',
+  name: '自定义插件输出',
   intro: '自定义配置外部输出，使用插件时，仅暴露自定义配置的输出',
   showStatus: false,
   inputs: [],

packages/global/core/workflow/template/system/tools.ts

@@ -27,7 +27,7 @@ export const ToolModule: FlowNodeTemplateType = {
   sourceHandle: getHandleConfig(true, true, false, true),
   targetHandle: getHandleConfig(true, true, false, true),
   avatar: '/imgs/workflow/tool.svg',
-  name: '工具调用（实验）',
+  name: '工具调用(实验)',
   intro: '通过AI模型自动选择一个或多个功能块进行调用，也可以对插件进行调用。',
   showStatus: true,
   inputs: [
@@ -64,5 +64,14 @@ export const ToolModule: FlowNodeTemplateType = {
     Input_Template_History,
     Input_Template_UserChatInput
   ],
-  outputs: []
+  outputs: [
+    {
+      id: NodeOutputKeyEnum.answerText,
+      key: NodeOutputKeyEnum.answerText,
+      label: 'core.module.output.label.Ai response content',
+      description: 'core.module.output.description.Ai response content',
+      valueType: WorkflowIOValueTypeEnum.string,
+      type: FlowNodeOutputTypeEnum.static
+    }
+  ]
 };

packages/global/core/workflow/template/system/variableUpdate/index.tsx

@@ -0,0 +1,42 @@
+import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../../node/constant';
+import { FlowNodeTemplateType } from '../../../type/index.d';
+import {
+  FlowNodeTemplateTypeEnum,
+  NodeInputKeyEnum,
+  WorkflowIOValueTypeEnum
+} from '../../../constants';
+import { getHandleConfig } from '../../utils';
+
+export const VariableUpdateNode: FlowNodeTemplateType = {
+  id: FlowNodeTypeEnum.variableUpdate,
+  templateType: FlowNodeTemplateTypeEnum.tools,
+  flowNodeType: FlowNodeTypeEnum.variableUpdate,
+  sourceHandle: getHandleConfig(true, true, true, true),
+  targetHandle: getHandleConfig(true, true, true, true),
+  avatar: '/imgs/workflow/variable.png',
+  name: '变量更新',
+  intro: '可以更新指定节点的输出值或更新全局变量',
+  showStatus: true,
+  isTool: false,
+  inputs: [
+    {
+      key: NodeInputKeyEnum.updateList,
+      valueType: WorkflowIOValueTypeEnum.any,
+      label: '',
+      renderTypeList: [FlowNodeInputTypeEnum.hidden],
+      editField: {
+        key: true,
+        valueType: true
+      },
+      value: [
+        {
+          variable: ['', ''],
+          value: ['', ''],
+          valueType: WorkflowIOValueTypeEnum.string,
+          renderType: FlowNodeInputTypeEnum.input
+        }
+      ]
+    }
+  ],
+  outputs: []
+};

packages/global/core/workflow/template/system/variableUpdate/type.d.ts

@@ -0,0 +1,10 @@
+import { FlowNodeInputTypeEnum } from '../../../node/constant';
+import { ReferenceValueProps } from '../../..//type/io';
+import { WorkflowIOValueTypeEnum } from '../../../constants';
+
+export type TUpdateListItem = {
+  variable?: ReferenceValueProps;
+  value: ReferenceValueProps;
+  valueType?: WorkflowIOValueTypeEnum;
+  renderType: FlowNodeInputTypeEnum.input | FlowNodeInputTypeEnum.reference;
+};

packages/global/core/workflow/type/index.d.ts

@@ -40,7 +40,7 @@ export type FlowNodeCommonType = {
 };
 
 export type FlowNodeTemplateType = FlowNodeCommonType & {
-  id: string; // module id, unique
+  id: string; // node id, unique
   templateType: `${FlowNodeTemplateTypeEnum}`;
 
   // show handle
@@ -132,11 +132,12 @@ export type ChatDispatchProps = {
   chatId?: string;
   responseChatItemId?: string;
   histories: ChatItemType[];
-  variables: Record<string, any>;
+  variables: Record<string, any>; // global variable
-  inputFiles?: UserChatItemValueItemType['file'][];
+  query: UserChatItemValueItemType[]; // trigger query
   stream: boolean;
   detail: boolean; // response detail
   maxRunTimes: number;
+  isToolCall?: boolean;
 };
 
 export type ModuleDispatchProps<T> = ChatDispatchProps & {

packages/global/core/workflow/utils.ts

@@ -15,6 +15,7 @@ import type {
 } from '../app/type';
 import { EditorVariablePickerType } from '../../../web/components/common/Textarea/PromptEditor/type';
 import { defaultWhisperConfig } from '../app/constants';
+import { IfElseResultEnum } from './template/system/ifElse/constant';
 
 export const getHandleId = (nodeId: string, type: 'source' | 'target', key: string) => {
   return `${nodeId}-${type}-${key}`;
@@ -35,13 +36,17 @@ export const checkInputIsReference = (input: FlowNodeInputItemType) => {
 
 /* node  */
 export const getGuideModule = (modules: StoreNodeItemType[]) =>
-  modules.find((item) => item.flowNodeType === FlowNodeTypeEnum.systemConfig);
+  modules.find(
-
+    (item) =>
+      item.flowNodeType === FlowNodeTypeEnum.systemConfig ||
+      // @ts-ignore (adapt v1)
+      item.flowType === FlowNodeTypeEnum.systemConfig
+  );
 export const splitGuideModule = (guideModules?: StoreNodeItemType) => {
   const welcomeText: string =
     guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.welcomeText)?.value || '';
 
-  const variableModules: VariableItemType[] =
+  const variableNodes: VariableItemType[] =
     guideModules?.inputs.find((item) => item.key === NodeInputKeyEnum.variables)?.value || [];
 
   const questionGuide: boolean =
@@ -62,13 +67,43 @@ export const splitGuideModule = (guideModules?: StoreNodeItemType) => {
 
   return {
     welcomeText,
-    variableModules,
+    variableNodes,
     questionGuide,
     ttsConfig,
     whisperConfig,
     scheduledTriggerConfig
   };
 };
+export const replaceAppChatConfig = ({
+  node,
+  variableList,
+  welcomeText
+}: {
+  node?: StoreNodeItemType;
+  variableList?: VariableItemType[];
+  welcomeText?: string;
+}): StoreNodeItemType | undefined => {
+  if (!node) return;
+  return {
+    ...node,
+    inputs: node.inputs.map((input) => {
+      if (input.key === NodeInputKeyEnum.variables && variableList) {
+        return {
+          ...input,
+          value: variableList
+        };
+      }
+      if (input.key === NodeInputKeyEnum.welcomeText && welcomeText) {
+        return {
+          ...input,
+          value: welcomeText
+        };
+      }
+
+      return input;
+    })
+  };
+};
 
 export const getOrInitModuleInputValue = (input: FlowNodeInputItemType) => {
   if (input.value !== undefined || !input.valueType) return input.value;
@@ -132,3 +167,11 @@ export const formatEditorVariablePickerIcon = (
     icon: item.type ? variableMap[item.type]?.icon : variableMap['input'].icon
   }));
 };
+
+export const isReferenceValue = (value: any): boolean => {
+  return Array.isArray(value) && value.length === 2 && typeof value[0] === 'string';
+};
+
+export const getElseIFLabel = (i: number) => {
+  return i === 0 ? IfElseResultEnum.IF : `${IfElseResultEnum.ELSE_IF} ${i}`;
+};

packages/global/support/outLink/constant.ts

@@ -1,5 +1,6 @@
-export enum OutLinkTypeEnum {
+export enum PublishChannelEnum {
   share = 'share',
   iframe = 'iframe',
-  apikey = 'apikey'
+  apikey = 'apikey',
+  feishu = 'feishu'
 }

packages/global/support/outLink/type.d.ts

@@ -1,31 +1,79 @@
 import { AppSchema } from 'core/app/type';
-import { OutLinkTypeEnum } from './constant';
+import { PublishChannelEnum } from './constant';
 
-export type OutLinkSchema = {
+// Feishu Config interface
+export interface FeishuType {
+  appId: string;
+  appSecret: string;
+  // Encrypt config
+  // refer to: https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/configure-encrypt-key
+  encryptKey?: string; // no secret if null
+  // Token Verification
+  // refer to: https://open.feishu.cn/document/server-docs/event-subscription-guide/event-subscription-configure-/encrypt-key-encryption-configuration-case
+  verificationToken: string;
+}
+
+// TODO: Unused
+export interface WecomType {
+  ReplyLimit: Boolean;
+  defaultResponse: string;
+  immediateResponse: boolean;
+  WXWORK_TOKEN: string;
+  WXWORK_AESKEY: string;
+  WXWORK_SECRET: string;
+  WXWORD_ID: string;
+}
+
+export type OutLinkSchema<T = void> = {
   _id: string;
   shareId: string;
   teamId: string;
   tmbId: string;
   appId: string;
+  // teamId: Schema.Types.ObjectId;
+  // tmbId: Schema.Types.ObjectId;
+  // appId: Schema.Types.ObjectId;
   name: string;
   usagePoints: number;
   lastTime: Date;
-  type: `${OutLinkTypeEnum}`;
+  type: PublishChannelEnum;
+
+  // whether the response content is detailed
   responseDetail: boolean;
+
+  // response when request
+  immediateResponse?: string;
+  // response when error or other situation
+  defaultResponse?: string;
+
   limit?: {
     expiredTime?: Date;
+    // Questions per minute
     QPM: number;
     maxUsagePoints: number;
+    // Verification message hook url
     hookUrl?: string;
   };
+
+  app?: T;
 };
+
+// to handle MongoDB querying
 export type OutLinkWithAppType = Omit<OutLinkSchema, 'appId'> & {
   appId: AppSchema;
 };
 
-export type OutLinkEditType = {
+// Edit the Outlink
+export type OutLinkEditType<T = void> = {
   _id?: string;
   name: string;
-  responseDetail: OutLinkSchema['responseDetail'];
+  responseDetail: OutLinkSchema<T>['responseDetail'];
-  limit: OutLinkSchema['limit'];
+  // response when request
+  immediateResponse?: string;
+  // response when error or other situation
+  defaultResponse?: string;
+  limit?: OutLinkSchema<T>['limit'];
+
+  // config for specific platform
+  app?: T;
 };

packages/service/common/buffer/rawText/schema.ts

@@ -28,6 +28,6 @@ try {
   console.log(error);
 }
 
-export const MongoRwaTextBuffer: Model<RawTextBufferSchemaType> =
+export const MongoRawTextBuffer: Model<RawTextBufferSchemaType> =
   models[collectionName] || model(collectionName, RawTextBufferSchema);
-MongoRwaTextBuffer.syncIndexes();
+MongoRawTextBuffer.syncIndexes();

packages/service/common/file/gridfs/controller.ts

@@ -6,8 +6,7 @@ import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
 import { MongoFileSchema } from './schema';
 import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
-import { ReadFileByBufferParams } from '../read/type';
+import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
-import { MongoRwaTextBuffer } from '../../buffer/rawText/schema';
 import { readFileRawContent } from '../read/utils';
 import { PassThrough } from 'stream';
 
@@ -163,7 +162,7 @@ export const readFileContentFromMongo = async ({
   filename: string;
 }> => {
   // read buffer
-  const fileBuffer = await MongoRwaTextBuffer.findOne({ sourceId: fileId }).lean();
+  const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: fileId }).lean();
   if (fileBuffer) {
     return {
       rawText: fileBuffer.rawText,
@@ -197,23 +196,19 @@ export const readFileContentFromMongo = async ({
     });
   })();
 
-  const params: ReadFileByBufferParams = {
+  const { rawText } = await readFileRawContent({
+    extension,
+    csvFormat,
     teamId,
     buffer: fileBuffers,
     encoding,
     metadata: {
       relatedId: fileId
     }
-  };
-
-  const { rawText } = await readFileRawContent({
-    extension,
-    csvFormat,
-    params
   });
 
   if (rawText.trim()) {
-    MongoRwaTextBuffer.create({
+    MongoRawTextBuffer.create({
       sourceId: fileId,
       rawText,
       metadata: {

packages/service/common/file/read/html.ts

@@ -1,23 +0,0 @@
-import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
-import { initMarkdownText } from './utils';
-import { htmlToMarkdown } from '../../string/markdown';
-import { readFileRawText } from './rawText';
-
-export const readHtmlRawText = async (
-  params: ReadFileByBufferParams
-): Promise<ReadFileResponse> => {
-  const { teamId, metadata } = params;
-  const { rawText: html } = readFileRawText(params);
-
-  const md = await htmlToMarkdown(html);
-
-  const rawText = await initMarkdownText({
-    teamId,
-    md,
-    metadata
-  });
-
-  return {
-    rawText
-  };
-};

packages/service/common/file/read/markdown.ts

@@ -1,18 +0,0 @@
-import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
-import { initMarkdownText } from './utils';
-import { readFileRawText } from './rawText';
-
-export const readMarkdown = async (params: ReadFileByBufferParams): Promise<ReadFileResponse> => {
-  const { teamId, metadata } = params;
-  const { rawText: md } = readFileRawText(params);
-
-  const rawText = await initMarkdownText({
-    teamId,
-    md,
-    metadata
-  });
-
-  return {
-    rawText
-  };
-};

packages/service/common/file/read/type.d.ts

@@ -1,12 +0,0 @@
-export type ReadFileByBufferParams = {
-  teamId: string;
-  buffer: Buffer;
-  encoding: string;
-  metadata?: Record<string, any>;
-};
-
-export type ReadFileResponse = {
-  rawText: string;
-  formatText?: string;
-  metadata?: Record<string, any>;
-};

packages/service/common/file/read/utils.ts

@@ -1,16 +1,10 @@
-import { markdownProcess } from '@fastgpt/global/common/string/markdown';
+import { markdownProcess, simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
 import { uploadMongoImg } from '../image/controller';
 import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
 import { addHours } from 'date-fns';
-import { ReadFileByBufferParams } from './type';
+
-import { readFileRawText } from '../read/rawText';
+import { WorkerNameEnum, runWorker } from '../../../worker/utils';
-import { readMarkdown } from '../read/markdown';
+import { ReadFileResponse } from '../../../worker/file/type';
-import { readHtmlRawText } from '../read/html';
-import { readPdfFile } from '../read/pdf';
-import { readWordFile } from '../read/word';
-import { readCsvRawText } from '../read/csv';
-import { readPptxRawText } from '../read/pptx';
-import { readXlsxRawText } from '../read/xlsx';
 
 export const initMarkdownText = ({
   teamId,
@@ -36,46 +30,39 @@ export const initMarkdownText = ({
 export const readFileRawContent = async ({
   extension,
   csvFormat,
-  params
+  teamId,
+  buffer,
+  encoding,
+  metadata
 }: {
   csvFormat?: boolean;
   extension: string;
-  params: ReadFileByBufferParams;
+  teamId: string;
+  buffer: Buffer;
+  encoding: string;
+  metadata?: Record<string, any>;
 }) => {
-  switch (extension) {
+  const result = await runWorker<ReadFileResponse>(WorkerNameEnum.readFile, {
-    case 'txt':
+    extension,
-      return readFileRawText(params);
+    csvFormat,
-    case 'md':
+    encoding,
-      return readMarkdown(params);
+    buffer
-    case 'html':
+  });
-      return readHtmlRawText(params);
+
-    case 'pdf':
+  // markdown data format
-      return readPdfFile(params);
+  if (['md', 'html', 'docx'].includes(extension)) {
-    case 'docx':
+    result.rawText = await initMarkdownText({
-      return readWordFile(params);
+      teamId: teamId,
-    case 'pptx':
+      md: result.rawText,
-      return readPptxRawText(params);
+      metadata: metadata
-    case 'xlsx':
+    });
-      const xlsxResult = await readXlsxRawText(params);
-      if (csvFormat) {
-        return {
-          rawText: xlsxResult.formatText || ''
-        };
-      }
-      return {
-        rawText: xlsxResult.rawText
-      };
-    case 'csv':
-      const csvResult = await readCsvRawText(params);
-      if (csvFormat) {
-        return {
-          rawText: csvResult.formatText || ''
-        };
-      }
-      return {
-        rawText: csvResult.rawText
-      };
-    default:
-      return Promise.reject('Only support .txt, .md, .html, .pdf, .docx, pptx, .csv, .xlsx');
   }
+
+  return result;
+};
+
+export const htmlToMarkdown = async (html?: string | null) => {
+  const md = await runWorker<string>(WorkerNameEnum.htmlStr2Md, { html: html || '' });
+
+  return simpleMarkdownText(md);
 };

packages/service/common/file/read/word.ts

@@ -1,35 +0,0 @@
-import mammoth from 'mammoth';
-import { htmlToMarkdown } from '../../string/markdown';
-import { ReadFileByBufferParams, ReadFileResponse } from './type';
-import { initMarkdownText } from './utils';
-
-/**
- * read docx to markdown
- */
-export const readWordFile = async ({
-  teamId,
-  buffer,
-  metadata = {}
-}: ReadFileByBufferParams): Promise<ReadFileResponse> => {
-  try {
-    const { value: html } = await mammoth.convertToHtml({
-      buffer
-    });
-
-    const md = await htmlToMarkdown(html);
-
-    const rawText = await initMarkdownText({
-      teamId,
-      md,
-      metadata
-    });
-
-    return {
-      rawText,
-      metadata: {}
-    };
-  } catch (error) {
-    console.log('error doc read:', error);
-    return Promise.reject('Can not read doc file, please convert to PDF');
-  }
-};

packages/service/common/middle/cors.ts

@@ -1,19 +1,12 @@
-import type { NextApiResponse, NextApiHandler, NextApiRequest } from 'next';
+import type { NextApiResponse, NextApiRequest } from 'next';
 import NextCors from 'nextjs-cors';
 
-export function withNextCors(handler: NextApiHandler): NextApiHandler {
+export async function withNextCors(req: NextApiRequest, res: NextApiResponse) {
-  return async function nextApiHandlerWrappedWithNextCors(
+  const methods = ['GET', 'eHEAD', 'PUT', 'PATCH', 'POST', 'DELETE'];
-    req: NextApiRequest,
+  const origin = req.headers.origin;
-    res: NextApiResponse
+  await NextCors(req, res, {
-  ) {
+    methods,
-    const methods = ['GET', 'eHEAD', 'PUT', 'PATCH', 'POST', 'DELETE'];
+    origin: origin,
-    const origin = req.headers.origin;
+    optionsSuccessStatus: 200
-    await NextCors(req, res, {
+  });
-      methods,
-      origin: origin,
-      optionsSuccessStatus: 200
-    });
-
-    return handler(req, res);
-  };
 }

packages/service/common/response/index.ts

@@ -18,9 +18,10 @@ export const jsonRes = <T = any>(
     message?: string;
     data?: T;
     error?: any;
+    url?: string;
   }
 ) => {
-  const { code = 200, message = '', data = null, error } = props || {};
+  const { code = 200, message = '', data = null, error, url } = props || {};
 
   const errResponseKey = typeof error === 'string' ? error : error?.message;
   // Specified error
@@ -47,7 +48,7 @@ export const jsonRes = <T = any>(
       msg = error?.error?.message;
     }
 
-    addLog.error(`response error: ${msg}`, error);
+    addLog.error(`Api response error: ${url}, ${msg}`, error);
   }
 
   res.status(code).json({

packages/service/common/string/cheerio.ts

@@ -1,7 +1,7 @@
 import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api';
 import * as cheerio from 'cheerio';
 import axios from 'axios';
-import { htmlToMarkdown } from './markdown';
+import { htmlToMarkdown } from '../file/read/utils';
 
 export const cheerioToHtml = ({
   fetchUrl,
@@ -77,7 +77,9 @@ export const urlsFetch = async ({
           $,
           selector
         });
+        console.log('html====', html);
         const md = await htmlToMarkdown(html);
+        console.log('html====', md);
 
         return {
           url,

packages/service/common/string/markdown.ts

@@ -1,9 +0,0 @@
-import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown';
-import { WorkerNameEnum, runWorker } from '../../worker/utils';
-
-/* html string to markdown */
-export const htmlToMarkdown = async (html?: string | null) => {
-  const md = await runWorker<string>(WorkerNameEnum.htmlStr2Md, { html: html || '' });
-
-  return simpleMarkdownText(md);
-};

packages/service/common/vectorStore/pg/controller.ts

@@ -23,7 +23,7 @@ export async function initPg() {
     `);
 
     await PgClient.query(
-      `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 64);`
+      `CREATE INDEX CONCURRENTLY IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 32, ef_construction = 100);`
     );
     await PgClient.query(
       `CREATE INDEX CONCURRENTLY IF NOT EXISTS team_dataset_collection_index ON ${PgDatasetTableName} USING btree(team_id, dataset_id, collection_id);`

packages/service/common/vectorStore/pg/index.ts

@@ -169,7 +169,16 @@ class PgClass {
   }
   async query<T extends QueryResultRow = any>(sql: string) {
     const pg = await connectPg();
-    return pg.query<T>(sql);
+    const start = Date.now();
+    return pg.query<T>(sql).then((res) => {
+      const time = Date.now() - start;
+
+      if (time > 300) {
+        addLog.warn(`pg query time: ${time}ms, sql: ${sql}`);
+      }
+
+      return res;
+    });
   }
 }
 

packages/service/core/ai/rerank/index.ts

@@ -1,3 +1,4 @@
+import { addLog } from '../../../common/system/log';
 import { POST } from '../../../common/api/serverRequest';
 
 type PostReRankResponse = {
@@ -38,7 +39,11 @@ export function reRankRecall({
     }
   )
     .then((data) => {
-      console.log('rerank time:', Date.now() - start);
+      addLog.info('ReRank finish:', { time: Date.now() - start });
+
+      if (!data?.results || data?.results?.length === 0) {
+        addLog.error('ReRank error, empty result', data);
+      }
 
       return data?.results?.map((item) => ({
         id: documents[item.index].id,
@@ -46,7 +51,7 @@ export function reRankRecall({
       }));
     })
     .catch((err) => {
-      console.log('rerank error:', err);
+      addLog.error('rerank error', err);
 
       return [];
     });

packages/service/core/app/controller.ts

@@ -0,0 +1,65 @@
+import { AppSchema } from '@fastgpt/global/core/app/type';
+import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
+import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
+import { getLLMModel } from '../ai/model';
+import { MongoAppVersion } from './versionSchema';
+
+export const beforeUpdateAppFormat = <T extends AppSchema['modules'] | undefined>({
+  nodes
+}: {
+  nodes: T;
+}) => {
+  if (nodes) {
+    let maxTokens = 3000;
+
+    nodes.forEach((item) => {
+      if (
+        item.flowNodeType === FlowNodeTypeEnum.chatNode ||
+        item.flowNodeType === FlowNodeTypeEnum.tools
+      ) {
+        const model =
+          item.inputs.find((item) => item.key === NodeInputKeyEnum.aiModel)?.value || '';
+        const chatModel = getLLMModel(model);
+        const quoteMaxToken = chatModel.quoteMaxToken || 3000;
+
+        maxTokens = Math.max(maxTokens, quoteMaxToken);
+      }
+    });
+
+    nodes.forEach((item) => {
+      if (item.flowNodeType === FlowNodeTypeEnum.datasetSearchNode) {
+        item.inputs.forEach((input) => {
+          if (input.key === NodeInputKeyEnum.datasetMaxTokens) {
+            const val = input.value as number;
+            if (val > maxTokens) {
+              input.value = maxTokens;
+            }
+          }
+        });
+      }
+    });
+  }
+
+  return {
+    nodes
+  };
+};
+
+export const getAppLatestVersion = async (appId: string, app?: AppSchema) => {
+  const version = await MongoAppVersion.findOne({
+    appId
+  }).sort({
+    time: -1
+  });
+
+  if (version) {
+    return {
+      nodes: version.nodes,
+      edges: version.edges
+    };
+  }
+  return {
+    nodes: app?.modules || [],
+    edges: app?.edges || []
+  };
+};

packages/service/core/app/schema.ts

@@ -8,7 +8,7 @@ import {
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-export const appCollectionName = 'apps';
+export const AppCollectionName = 'apps';
 
 const AppSchema = new Schema({
   teamId: {
@@ -46,6 +46,8 @@ const AppSchema = new Schema({
     type: Date,
     default: () => new Date()
   },
+
+  // tmp store
   modules: {
     type: Array,
     default: []
@@ -92,6 +94,6 @@ try {
 }
 
 export const MongoApp: Model<AppType> =
-  models[appCollectionName] || model(appCollectionName, AppSchema);
+  models[AppCollectionName] || model(AppCollectionName, AppSchema);
 
 MongoApp.syncIndexes();

packages/service/core/app/versionSchema.ts

@@ -0,0 +1,36 @@
+import { connectionMongo, type Model } from '../../common/mongo';
+const { Schema, model, models } = connectionMongo;
+import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
+
+export const AppVersionCollectionName = 'app.versions';
+
+const AppVersionSchema = new Schema({
+  appId: {
+    type: Schema.Types.ObjectId,
+    ref: AppVersionCollectionName,
+    required: true
+  },
+  time: {
+    type: Date,
+    default: () => new Date()
+  },
+  nodes: {
+    type: Array,
+    default: []
+  },
+  edges: {
+    type: Array,
+    default: []
+  }
+});
+
+try {
+  AppVersionSchema.index({ appId: 1, time: -1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoAppVersion: Model<AppVersionSchemaType> =
+  models[AppVersionCollectionName] || model(AppVersionCollectionName, AppVersionSchema);
+
+MongoAppVersion.syncIndexes();

packages/service/core/chat/chatItemSchema.ts

@@ -7,7 +7,7 @@ import {
   TeamCollectionName,
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
-import { appCollectionName } from '../app/schema';
+import { AppCollectionName } from '../app/schema';
 import { userCollectionName } from '../../support/user/schema';
 import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
@@ -40,7 +40,7 @@ const ChatItemSchema = new Schema({
   },
   appId: {
     type: Schema.Types.ObjectId,
-    ref: appCollectionName,
+    ref: AppCollectionName,
     required: true
   },
   time: {

packages/service/core/chat/chatSchema.ts

@@ -6,7 +6,7 @@ import {
   TeamCollectionName,
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
-import { appCollectionName } from '../app/schema';
+import { AppCollectionName } from '../app/schema';
 
 export const chatCollectionName = 'chat';
 
@@ -31,7 +31,7 @@ const ChatSchema = new Schema({
   },
   appId: {
     type: Schema.Types.ObjectId,
-    ref: appCollectionName,
+    ref: AppCollectionName,
     required: true
   },
   updateTime: {
@@ -61,7 +61,15 @@ const ChatSchema = new Schema({
   outLinkUid: {
     type: String
   },
+
+  variableList: {
+    type: Array
+  },
+  welcomeText: {
+    type: String
+  },
   variables: {
+    // variable value
     type: Object,
     default: {}
   },

packages/service/core/plugin/type.d.ts

@@ -1,5 +1,6 @@
 import { PluginTemplateType } from '@fastgpt/global/core/plugin/type.d';
 
 declare global {
+  var communityPluginsV1: PluginTemplateType[];
   var communityPlugins: PluginTemplateType[];
 }

packages/service/core/workflow/dispatch/agent/classifyQuestion.ts

@@ -131,7 +131,9 @@ const completions = async ({
   console.log(answer, '----');
 
   const id =
-    agents.find((item) => answer.includes(item.key) || answer.includes(item.value))?.key || '';
+    agents.find((item) => answer.includes(item.key))?.key ||
+    agents.find((item) => answer.includes(item.value))?.key ||
+    '';
 
   return {
     tokens: await countMessagesTokens(messages),

packages/service/core/workflow/dispatch/agent/runTool/functionCall.ts

@@ -161,6 +161,7 @@ export const runToolWithFunctionCall = async (
 
         const toolRunResponse = await dispatchWorkFlow({
           ...props,
+          isToolCall: true,
           runtimeNodes: runtimeNodes.map((item) =>
             item.nodeId === toolNode.nodeId
               ? {

packages/service/core/workflow/dispatch/agent/runTool/index.ts

@@ -23,7 +23,9 @@ import { runToolWithPromptCall } from './promptCall';
 import { replaceVariable } from '@fastgpt/global/common/string/tools';
 import { Prompt_Tool_Call } from './constants';
 
-type Response = DispatchNodeResultType<{}>;
+type Response = DispatchNodeResultType<{
+  [NodeOutputKeyEnum.answerText]: string;
+}>;
 
 export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<Response> => {
   const {
@@ -129,6 +131,10 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
   const flatUsages = dispatchFlowResponse.map((item) => item.flowUsages).flat();
 
   return {
+    [NodeOutputKeyEnum.answerText]: assistantResponses
+      .filter((item) => item.text?.content)
+      .map((item) => item.text?.content || '')
+      .join(''),
     [DispatchNodeResponseKeyEnum.assistantResponses]: assistantResponses,
     [DispatchNodeResponseKeyEnum.nodeResponse]: {
       totalPoints: totalPointsUsage,

packages/service/core/workflow/dispatch/agent/runTool/promptCall.ts

@@ -185,6 +185,7 @@ export const runToolWithPromptCall = async (
 
     const moduleRunResponse = await dispatchWorkFlow({
       ...props,
+      isToolCall: true,
       runtimeNodes: runtimeNodes.map((item) =>
         item.nodeId === toolNode.nodeId
           ? {

packages/service/core/workflow/dispatch/agent/runTool/toolChoice.ts

@@ -182,6 +182,7 @@ export const runToolWithToolChoice = async (
 
         const toolRunResponse = await dispatchWorkFlow({
           ...props,
+          isToolCall: true,
           runtimeNodes: runtimeNodes.map((item) =>
             item.nodeId === toolNode.nodeId
               ? {

packages/service/core/workflow/dispatch/chat/oneapi.ts

@@ -25,6 +25,7 @@ import {
 } from '../../../../common/string/tiktoken/index';
 import {
   chats2GPTMessages,
+  chatValue2RuntimePrompt,
   getSystemPrompt,
   GPTMessages2Chats,
   runtimePrompt2ChatsValue
@@ -66,7 +67,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
     user,
     histories,
     node: { name },
-    inputFiles = [],
+    query,
     params: {
       model,
       temperature = 0,
@@ -80,6 +81,8 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise<ChatResp
       quotePrompt
     }
   } = props;
+  const { files: inputFiles } = chatValue2RuntimePrompt(query);
+
   if (!userChatInput && inputFiles.length === 0) {
     return Promise.reject('Question is empty');
   }

packages/service/core/workflow/dispatch/index.ts

@@ -42,7 +42,8 @@ import { dispatchLafRequest } from './tools/runLaf';
 import { dispatchIfElse } from './tools/runIfElse';
 import { RuntimeEdgeItemType } from '@fastgpt/global/core/workflow/type/edge';
 import { getReferenceVariableValue } from '@fastgpt/global/core/workflow/runtime/utils';
-import { dispatchSystemConfig } from './init/systemConfiig';
+import { dispatchSystemConfig } from './init/systemConfig';
+import { dispatchUpdateVariable } from './tools/runUpdateVar';
 
 const callbackMap: Record<`${FlowNodeTypeEnum}`, Function> = {
   [FlowNodeTypeEnum.workflowStart]: dispatchWorkflowStart,
@@ -62,6 +63,7 @@ const callbackMap: Record<`${FlowNodeTypeEnum}`, Function> = {
   [FlowNodeTypeEnum.stopTool]: dispatchStopToolCall,
   [FlowNodeTypeEnum.lafModule]: dispatchLafRequest,
   [FlowNodeTypeEnum.ifElseNode]: dispatchIfElse,
+  [FlowNodeTypeEnum.variableUpdate]: dispatchUpdateVariable,
 
   // none
   [FlowNodeTypeEnum.systemConfig]: dispatchSystemConfig,
@@ -69,21 +71,25 @@ const callbackMap: Record<`${FlowNodeTypeEnum}`, Function> = {
   [FlowNodeTypeEnum.globalVariable]: () => Promise.resolve()
 };
 
-/* running */
+type Props = ChatDispatchProps & {
-export async function dispatchWorkFlow({
-  res,
-  runtimeNodes = [],
-  runtimeEdges = [],
-  histories = [],
-  variables = {},
-  user,
-  stream = false,
-  detail = false,
-  ...props
-}: ChatDispatchProps & {
   runtimeNodes: RuntimeNodeItemType[];
   runtimeEdges: RuntimeEdgeItemType[];
-}): Promise<DispatchFlowResponse> {
+};
+
+/* running */
+export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowResponse> {
+  let {
+    res,
+    runtimeNodes = [],
+    runtimeEdges = [],
+    histories = [],
+    variables = {},
+    user,
+    stream = false,
+    detail = false,
+    ...props
+  } = data;
+
   // set sse response headers
   if (stream && res) {
     res.setHeader('Content-Type', 'text/event-stream;charset=utf-8');
@@ -93,7 +99,7 @@ export async function dispatchWorkFlow({
   }
 
   variables = {
-    ...getSystemVariable({ timezone: user.timezone }),
+    ...getSystemVariable(data),
     ...variables
   };
 
@@ -142,10 +148,8 @@ export async function dispatchWorkFlow({
     }
     if (assistantResponses) {
       chatAssistantResponse = chatAssistantResponse.concat(assistantResponses);
-    }
+    } else if (answerText) {
-
+      // save assistant text response
-    // save assistant text response
-    if (answerText) {
       const isResponseAnswerText =
         inputs.find((item) => item.key === NodeInputKeyEnum.aiChatIsResponseText)?.value ?? true;
       if (isResponseAnswerText) {
@@ -220,9 +224,16 @@ export async function dispatchWorkFlow({
     ).then((result) => {
       const flat = result.flat();
       if (flat.length === 0) return;
-      // update output
+
+      // Update the node output at the end of the run and get the next nodes
       const nextNodes = flat.map((item) => nodeOutput(item.node, item.result)).flat();
-      return checkNodeCanRun(nextNodes);
+
+      // Remove repeat nodes(Make sure that the node is only executed once)
+      const filterNextNodes = nextNodes.filter(
+        (node, index, self) => self.findIndex((t) => t.nodeId === node.nodeId) === index
+      );
+
+      return checkNodeCanRun(filterNextNodes);
     });
   }
   // 运行完一轮后，清除连线的状态，避免污染进程
@@ -278,7 +289,8 @@ export async function dispatchWorkFlow({
       node,
       runtimeNodes,
       runtimeEdges,
-      params
+      params,
+      mode: props.mode === 'debug' ? 'test' : props.mode
     };
 
     // run module
@@ -357,7 +369,8 @@ export async function dispatchWorkFlow({
     },
     [DispatchNodeResponseKeyEnum.assistantResponses]:
       mergeAssistantResponseAnswerText(chatAssistantResponse),
-    [DispatchNodeResponseKeyEnum.toolResponses]: toolRunResponse
+    [DispatchNodeResponseKeyEnum.toolResponses]: toolRunResponse,
+    newVariables: removeSystemVariable(variables)
   };
 }
 
@@ -379,12 +392,34 @@ export function responseStatus({
 }
 
 /* get system variable */
-export function getSystemVariable({ timezone }: { timezone: string }) {
+export function getSystemVariable({
+  user,
+  appId,
+  chatId,
+  responseChatItemId,
+  histories = []
+}: Props) {
   return {
-    cTime: getSystemTime(timezone)
+    appId,
+    chatId,
+    responseChatItemId,
+    histories,
+    cTime: getSystemTime(user.timezone)
   };
 }
 
+/* remove system variable */
+const removeSystemVariable = (variables: Record<string, any>) => {
+  const copyVariables = { ...variables };
+  delete copyVariables.appId;
+  delete copyVariables.chatId;
+  delete copyVariables.responseChatItemId;
+  delete copyVariables.histories;
+  delete copyVariables.cTime;
+
+  return copyVariables;
+};
+
 /* Merge consecutive text messages into one */
 export const mergeAssistantResponseAnswerText = (response: AIChatItemValueItemType[]) => {
   const result: AIChatItemValueItemType[] = [];

packages/service/core/workflow/dispatch/init/systemConfig.tsx

@@ -0,0 +1,4 @@
+export const dispatchSystemConfig = (props: Record<string, any>) => {
+  const { variables } = props;
+  return variables;
+};

packages/service/core/workflow/dispatch/init/systemConfiig.tsx

@@ -1,10 +0,0 @@
-import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
-import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type/index.d';
-export type UserChatInputProps = ModuleDispatchProps<{
-  [NodeInputKeyEnum.userChatInput]: string;
-}>;
-
-export const dispatchSystemConfig = (props: Record<string, any>) => {
-  const { variables } = props as UserChatInputProps;
-  return variables;
-};

packages/service/core/workflow/dispatch/init/workflowStart.tsx

@@ -1,15 +1,22 @@
+import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
+import { UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type/index.d';
 export type UserChatInputProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.userChatInput]: string;
+  [NodeInputKeyEnum.inputFiles]: UserChatItemValueItemType['file'][];
 }>;
 
 export const dispatchWorkflowStart = (props: Record<string, any>) => {
   const {
-    variables: { userChatInput },
+    query,
-    params: { userChatInput: query }
+    params: { userChatInput }
   } = props as UserChatInputProps;
+
+  const { text, files } = chatValue2RuntimePrompt(query);
+
   return {
-    userChatInput: query || userChatInput
+    [NodeInputKeyEnum.userChatInput]: text || userChatInput,
+    [NodeInputKeyEnum.inputFiles]: files
   };
 };

packages/service/core/workflow/dispatch/plugin/run.ts

@@ -1,7 +1,7 @@
 import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type/index.d';
 import { dispatchWorkFlow } from '../index';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
-import { NodeInputKeyEnum, WorkflowIOValueTypeEnum } from '@fastgpt/global/core/workflow/constants';
+import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import { getPluginRuntimeById } from '../../../plugin/controller';
 import { authPluginCanUse } from '../../../../support/permission/auth/plugin';

packages/service/core/workflow/dispatch/tools/answer.ts

@@ -19,24 +19,24 @@ export const dispatchAnswer = (props: Record<string, any>): AnswerResponse => {
     res,
     detail,
     stream,
-    node: { name },
     params: { text = '' }
   } = props as AnswerProps;
 
   const formatText = typeof text === 'string' ? text : JSON.stringify(text, null, 2);
+  const responseText = `\n${formatText}`;
 
   if (res && stream) {
     responseWrite({
       res,
       event: detail ? SseResponseEventEnum.fastAnswer : undefined,
       data: textAdaptGptResponse({
-        text: `\n${formatText}`
+        text: responseText
       })
     });
   }
 
   return {
-    [NodeOutputKeyEnum.answerText]: formatText,
+    [NodeOutputKeyEnum.answerText]: responseText,
     [DispatchNodeResponseKeyEnum.nodeResponse]: {
       textOutput: formatText
     }

packages/service/core/workflow/dispatch/tools/http468.ts

@@ -45,10 +45,12 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
     detail,
     appId,
     chatId,
+    stream,
     responseChatItemId,
     variables,
     node: { outputs },
     histories,
+    isToolCall,
     params: {
       system_httpMethod: httpMethod = 'POST',
       system_httpReqUrl: httpReqUrl,
@@ -131,7 +133,7 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
       results[key] = valueTypeFormat(formatResponse[key], output.valueType);
     }
 
-    if (typeof formatResponse[NodeOutputKeyEnum.answerText] === 'string') {
+    if (stream && typeof formatResponse[NodeOutputKeyEnum.answerText] === 'string') {
       responseWrite({
         res,
         event: detail ? SseResponseEventEnum.fastAnswer : undefined,
@@ -156,17 +158,21 @@ export const dispatchHttp468Request = async (props: HttpRequestProps): Promise<H
     };
   } catch (error) {
     addLog.error('Http request error', error);
-    return {
+
-      [NodeOutputKeyEnum.failed]: true,
+    if (isToolCall) {
-      [DispatchNodeResponseKeyEnum.nodeResponse]: {
+      return {
-        totalPoints: 0,
+        [NodeOutputKeyEnum.failed]: true,
-        params: Object.keys(params).length > 0 ? params : undefined,
+        [DispatchNodeResponseKeyEnum.nodeResponse]: {
-        body: Object.keys(requestBody).length > 0 ? requestBody : undefined,
+          totalPoints: 0,
-        headers: Object.keys(headers).length > 0 ? headers : undefined,
+          params: Object.keys(params).length > 0 ? params : undefined,
-        httpResult: { error: formatHttpError(error) }
+          body: Object.keys(requestBody).length > 0 ? requestBody : undefined,
-      },
+          headers: Object.keys(headers).length > 0 ? headers : undefined,
-      [NodeOutputKeyEnum.httpRawResponse]: getErrText(error)
+          httpResult: { error: formatHttpError(error) }
-    };
+        },
+        [NodeOutputKeyEnum.httpRawResponse]: getErrText(error)
+      };
+    }
+    return Promise.reject(error);
   }
 };
 

packages/service/core/workflow/dispatch/tools/runApp.ts

@@ -35,7 +35,7 @@ export const dispatchAppRequest = async (props: Props): Promise<Response> => {
     stream,
     detail,
     histories,
-    inputFiles,
+    query,
     params: { userChatInput, history, app }
   } = props;
   let start = Date.now();
@@ -71,7 +71,7 @@ export const dispatchAppRequest = async (props: Props): Promise<Response> => {
     runtimeNodes: storeNodes2RuntimeNodes(appData.modules, getDefaultEntryNodeIds(appData.modules)),
     runtimeEdges: initWorkflowEdgeStatus(appData.edges),
     histories: chatHistories,
-    inputFiles,
+    query,
     variables: {
       ...props.variables,
       userChatInput
@@ -81,10 +81,7 @@ export const dispatchAppRequest = async (props: Props): Promise<Response> => {
   const completeMessages = chatHistories.concat([
     {
       obj: ChatRoleEnum.Human,
-      value: runtimePrompt2ChatsValue({
+      value: query
-        files: inputFiles,
-        text: userChatInput
-      })
     },
     {
       obj: ChatRoleEnum.AI,

packages/service/core/workflow/dispatch/tools/runIfElse.ts

@@ -1,70 +1,141 @@
-import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
+import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
-import { VariableConditionEnum } from '@fastgpt/global/core/workflow/template/system/ifElse/constant';
 import {
+  IfElseResultEnum,
+  VariableConditionEnum
+} from '@fastgpt/global/core/workflow/template/system/ifElse/constant';
+import {
+  ConditionListItemType,
   IfElseConditionType,
   IfElseListItemType
 } from '@fastgpt/global/core/workflow/template/system/ifElse/type';
 import { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type';
-import { getHandleId } from '@fastgpt/global/core/workflow/utils';
+import { getElseIFLabel, getHandleId } from '@fastgpt/global/core/workflow/utils';
+import { getReferenceVariableValue } from '@fastgpt/global/core/workflow/runtime/utils';
 
 type Props = ModuleDispatchProps<{
   [NodeInputKeyEnum.condition]: IfElseConditionType;
   [NodeInputKeyEnum.ifElseList]: IfElseListItemType[];
 }>;
+type Response = DispatchNodeResultType<{
+  [NodeOutputKeyEnum.ifElseResult]: string;
+}>;
+
+function isEmpty(value: any) {
+  return (
+    // 检查未定义或null值
+    value === undefined ||
+    value === null ||
+    // 检查空字符串
+    (typeof value === 'string' && value.trim() === '') ||
+    // 检查NaN
+    (typeof value === 'number' && isNaN(value)) ||
+    // 检查空数组
+    (Array.isArray(value) && value.length === 0) ||
+    // 检查空对象
+    (typeof value === 'object' && Object.keys(value).length === 0)
+  );
+}
+
+function isInclude(value: any, target: any) {
+  if (Array.isArray(value)) {
+    return value.map((item: any) => String(item)).includes(target);
+  } else if (typeof value === 'string') {
+    return value.includes(target);
+  } else {
+    return false;
+  }
+}
 
 function checkCondition(condition: VariableConditionEnum, variableValue: any, value: string) {
   const operations = {
-    [VariableConditionEnum.isEmpty]: () => !variableValue,
+    [VariableConditionEnum.isEmpty]: () => isEmpty(variableValue),
-    [VariableConditionEnum.isNotEmpty]: () => !!variableValue,
+    [VariableConditionEnum.isNotEmpty]: () => !isEmpty(variableValue),
-    [VariableConditionEnum.equalTo]: () => variableValue === value,
+
-    [VariableConditionEnum.notEqual]: () => variableValue !== value,
+    [VariableConditionEnum.equalTo]: () => String(variableValue) === value,
-    [VariableConditionEnum.greaterThan]: () => variableValue > Number(value),
+    [VariableConditionEnum.notEqual]: () => String(variableValue) !== value,
-    [VariableConditionEnum.lessThan]: () => variableValue < Number(value),
+
-    [VariableConditionEnum.greaterThanOrEqualTo]: () => variableValue >= Number(value),
+    // number
-    [VariableConditionEnum.lessThanOrEqualTo]: () => variableValue <= Number(value),
+    [VariableConditionEnum.greaterThan]: () => Number(variableValue) > Number(value),
-    [VariableConditionEnum.include]: () => variableValue.includes(value),
+    [VariableConditionEnum.lessThan]: () => Number(variableValue) < Number(value),
-    [VariableConditionEnum.notInclude]: () => !variableValue.includes(value),
+    [VariableConditionEnum.greaterThanOrEqualTo]: () => Number(variableValue) >= Number(value),
-    [VariableConditionEnum.startWith]: () => variableValue.startsWith(value),
+    [VariableConditionEnum.lessThanOrEqualTo]: () => Number(variableValue) <= Number(value),
-    [VariableConditionEnum.endWith]: () => variableValue.endsWith(value),
+
-    [VariableConditionEnum.lengthEqualTo]: () => variableValue.length === Number(value),
+    // array or string
-    [VariableConditionEnum.lengthNotEqualTo]: () => variableValue.length !== Number(value),
+    [VariableConditionEnum.include]: () => isInclude(variableValue, value),
-    [VariableConditionEnum.lengthGreaterThan]: () => variableValue.length > Number(value),
+    [VariableConditionEnum.notInclude]: () => !isInclude(variableValue, value),
-    [VariableConditionEnum.lengthGreaterThanOrEqualTo]: () => variableValue.length >= Number(value),
+
-    [VariableConditionEnum.lengthLessThan]: () => variableValue.length < Number(value),
+    // string
-    [VariableConditionEnum.lengthLessThanOrEqualTo]: () => variableValue.length <= Number(value)
+    [VariableConditionEnum.startWith]: () => variableValue?.startsWith(value),
+    [VariableConditionEnum.endWith]: () => variableValue?.endsWith(value),
+
+    // array
+    [VariableConditionEnum.lengthEqualTo]: () => variableValue?.length === Number(value),
+    [VariableConditionEnum.lengthNotEqualTo]: () => variableValue?.length !== Number(value),
+    [VariableConditionEnum.lengthGreaterThan]: () => variableValue?.length > Number(value),
+    [VariableConditionEnum.lengthGreaterThanOrEqualTo]: () =>
+      variableValue?.length >= Number(value),
+    [VariableConditionEnum.lengthLessThan]: () => variableValue?.length < Number(value),
+    [VariableConditionEnum.lengthLessThanOrEqualTo]: () => variableValue?.length <= Number(value)
   };
 
   return (operations[condition] || (() => false))();
 }
 
-export const dispatchIfElse = async (props: Props): Promise<DispatchNodeResultType<{}>> => {
+function getResult(
-  const {
+  condition: IfElseConditionType,
-    params,
+  list: ConditionListItemType[],
-    runtimeNodes,
+  variables: any,
-    node: { nodeId }
+  runtimeNodes: any[]
-  } = props;
+) {
-  const { condition, ifElseList } = params;
+  const listResult = list.map((item) => {
-  const listResult = ifElseList.map((item) => {
     const { variable, condition: variableCondition, value } = item;
 
-    const variableValue = runtimeNodes
+    const variableValue = getReferenceVariableValue({
-      .find((node) => node.nodeId === variable[0])
+      value: variable,
-      ?.outputs?.find((item) => item.id === variable[1])?.value;
+      variables,
+      nodes: runtimeNodes
+    });
 
     return checkCondition(variableCondition as VariableConditionEnum, variableValue, value || '');
   });
 
-  const result = condition === 'AND' ? listResult.every(Boolean) : listResult.some(Boolean);
+  return condition === 'AND' ? listResult.every(Boolean) : listResult.some(Boolean);
+}
+
+export const dispatchIfElse = async (props: Props): Promise<Response> => {
+  const {
+    params,
+    runtimeNodes,
+    variables,
+    node: { nodeId }
+  } = props;
+  const { ifElseList } = params;
+
+  let res = IfElseResultEnum.ELSE as string;
+  for (let i = 0; i < ifElseList.length; i++) {
+    const item = ifElseList[i];
+    const result = getResult(item.condition, item.list, variables, runtimeNodes);
+    if (result) {
+      res = getElseIFLabel(i);
+      break;
+    }
+  }
+
+  const resArray = Array.from({ length: ifElseList.length + 1 }, (_, index) => {
+    const label = index < ifElseList.length ? getElseIFLabel(index) : IfElseResultEnum.ELSE;
+    return getHandleId(nodeId, 'source', label);
+  });
 
   return {
+    [NodeOutputKeyEnum.ifElseResult]: res,
     [DispatchNodeResponseKeyEnum.nodeResponse]: {
       totalPoints: 0,
-      ifElseResult: result ? 'IF' : 'ELSE'
+      ifElseResult: res
     },
-    [DispatchNodeResponseKeyEnum.skipHandleId]: result
+    [DispatchNodeResponseKeyEnum.skipHandleId]: resArray.filter(
-      ? [getHandleId(nodeId, 'source', 'ELSE')]
+      (item) => item !== getHandleId(nodeId, 'source', res)
-      : [getHandleId(nodeId, 'source', 'IF')]
+    )
   };
 };

packages/service/core/workflow/dispatch/tools/runUpdateVar.ts

@@ -0,0 +1,59 @@
+import { NodeInputKeyEnum, VARIABLE_NODE_ID } from '@fastgpt/global/core/workflow/constants';
+import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
+import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
+import { getReferenceVariableValue } from '@fastgpt/global/core/workflow/runtime/utils';
+import { TUpdateListItem } from '@fastgpt/global/core/workflow/template/system/variableUpdate/type';
+import { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type';
+import { valueTypeFormat } from '../utils';
+
+type Props = ModuleDispatchProps<{
+  [NodeInputKeyEnum.updateList]: TUpdateListItem[];
+}>;
+
+export const dispatchUpdateVariable = async (
+  props: Props
+): Promise<DispatchNodeResultType<any>> => {
+  const { params, variables, runtimeNodes } = props;
+
+  const { updateList } = params;
+  updateList.forEach((item) => {
+    const varNodeId = item.variable?.[0];
+    const varKey = item.variable?.[1];
+
+    if (!varNodeId || !varKey) {
+      return;
+    }
+
+    const value = (() => {
+      if (!item.value?.[0]) {
+        return valueTypeFormat(item.value?.[1], item.valueType);
+      } else {
+        return getReferenceVariableValue({
+          value: item.value,
+          variables,
+          nodes: runtimeNodes
+        });
+      }
+    })();
+
+    if (varNodeId === VARIABLE_NODE_ID) {
+      // update global variable
+      variables[varKey] = value;
+    } else {
+      runtimeNodes
+        .find((node) => node.nodeId === varNodeId)
+        ?.outputs?.find((output) => {
+          if (output.id === varKey) {
+            output.value = value;
+            return true;
+          }
+        });
+    }
+  });
+
+  return {
+    [DispatchNodeResponseKeyEnum.nodeResponse]: {
+      totalPoints: 0
+    }
+  };
+};

packages/service/core/workflow/dispatch/type.d.ts

@@ -19,4 +19,5 @@ export type DispatchFlowResponse = {
   };
   [DispatchNodeResponseKeyEnum.toolResponses]: ToolRunResponseItemType;
   [DispatchNodeResponseKeyEnum.assistantResponses]: AIChatItemValueItemType[];
+  newVariables: Record<string, string>;
 };

packages/service/core/workflow/dispatch/utils.ts

@@ -47,7 +47,7 @@ export const filterToolNodeIdByEdges = ({
 
 export const getHistories = (history?: ChatItemType[] | number, histories: ChatItemType[] = []) => {
   if (!history) return [];
-  if (typeof history === 'number') return histories.slice(-history);
+  if (typeof history === 'number') return histories.slice(-(history * 2));
   if (Array.isArray(history)) return history;
 
   return [];

packages/service/core/workflow/dispatchV1/index.ts

@@ -279,7 +279,7 @@ export async function dispatchWorkFlowV1({
     )?.targets?.length;
 
     return moduleOutput(module, {
-      [NodeOutputKeyEnum.finish]: true,
+      finish: true,
       [NodeOutputKeyEnum.userChatInput]: hasUserChatInputTarget
         ? params[NodeOutputKeyEnum.userChatInput]
         : undefined,
@@ -295,7 +295,7 @@ export async function dispatchWorkFlowV1({
   modules.forEach((item) => {
     item.isEntry = false;
   });
-
+  // console.log(JSON.stringify(runningModules, null, 2));
   initModules.map((module) =>
     moduleInput(module, {
       ...startParams,

packages/service/core/workflow/dispatchV1/plugin/run.ts

@@ -1,13 +1,20 @@
 // @ts-nocheck
+
 import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type';
 import { dispatchWorkFlowV1 } from '../index';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
-import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
+import {
+  FlowNodeTemplateTypeEnum,
+  NodeInputKeyEnum
+} from '@fastgpt/global/core/workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
-import { getPluginRuntimeById } from '../../../plugin/controller';
+import { splitCombinePluginId } from '../../../plugin/controller';
 import { authPluginCanUse } from '../../../../support/permission/auth/plugin';
 import { setEntryEntries, DYNAMIC_INPUT_KEY } from '../utils';
 import { DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
+import { PluginRuntimeType, PluginTemplateType } from '@fastgpt/global/core/plugin/type';
+import { PluginSourceEnum } from '@fastgpt/global/core/plugin/constants';
+import { MongoPlugin } from '../../../plugin/schema';
 
 type RunPluginProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.pluginId]: string;
@@ -15,6 +22,45 @@ type RunPluginProps = ModuleDispatchProps<{
 }>;
 type RunPluginResponse = DispatchNodeResultType<{}>;
 
+const getPluginTemplateById = async (id: string): Promise<PluginTemplateType> => {
+  const { source, pluginId } = await splitCombinePluginId(id);
+  if (source === PluginSourceEnum.community) {
+    const item = global.communityPluginsV1?.find((plugin) => plugin.id === pluginId);
+
+    if (!item) return Promise.reject('plugin not found');
+
+    return item;
+  }
+  if (source === PluginSourceEnum.personal) {
+    const item = await MongoPlugin.findById(id).lean();
+    if (!item) return Promise.reject('plugin not found');
+    return {
+      id: String(item._id),
+      teamId: String(item.teamId),
+      name: item.name,
+      avatar: item.avatar,
+      intro: item.intro,
+      showStatus: true,
+      source: PluginSourceEnum.personal,
+      modules: item.modules,
+      templateType: FlowNodeTemplateTypeEnum.personalPlugin
+    };
+  }
+  return Promise.reject('plugin not found');
+};
+
+const getPluginRuntimeById = async (id: string): Promise<PluginRuntimeType> => {
+  const plugin = await getPluginTemplateById(id);
+
+  return {
+    teamId: plugin.teamId,
+    name: plugin.name,
+    avatar: plugin.avatar,
+    showStatus: plugin.showStatus,
+    modules: plugin.modules
+  };
+};
+
 export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPluginResponse> => {
   const {
     mode,
@@ -31,7 +77,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
   const plugin = await getPluginRuntimeById(pluginId);
 
   // concat dynamic inputs
-  const inputModule = plugin.nodes.find((item) => item.flowType === FlowNodeTypeEnum.pluginInput);
+  const inputModule = plugin.modules.find((item) => item.flowType === FlowNodeTypeEnum.pluginInput);
   if (!inputModule) return Promise.reject('Plugin error, It has no set input.');
   const hasDynamicInput = inputModule.inputs.find((input) => input.key === DYNAMIC_INPUT_KEY);
 
@@ -56,7 +102,7 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
 
   const { flowResponses, flowUsages, assistantResponses } = await dispatchWorkFlowV1({
     ...props,
-    modules: setEntryEntries(plugin.nodes).map((module) => ({
+    modules: setEntryEntries(plugin.modules).map((module) => ({
       ...module,
       showStatus: false
     })),

packages/service/core/workflow/dispatchV1/tools/answer.ts

@@ -32,6 +32,6 @@ export const dispatchAnswer = (props: Record<string, any>): AnswerResponse => {
   }
 
   return {
-    [NodeOutputKeyEnum.answerText]: formatText
+    [NodeOutputKeyEnum.answerText]: `\n${formatText}`
   };
 };

packages/service/core/workflow/dispatchV1/type.d.ts

@@ -12,4 +12,5 @@ export type DispatchFlowResponse = {
   flowUsages: ChatNodeUsageType[];
   [DispatchNodeResponseKeyEnum.toolResponses]: ToolRunResponseItemType;
   [DispatchNodeResponseKeyEnum.assistantResponses]: AIChatItemValueItemType[];
+  newVariables: Record<string, string>;
 };

packages/service/support/outLink/schema.ts

@@ -1,12 +1,11 @@
 import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { OutLinkSchema as SchemaType } from '@fastgpt/global/support/outLink/type';
-import { OutLinkTypeEnum } from '@fastgpt/global/support/outLink/constant';
 import {
   TeamCollectionName,
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
-import { appCollectionName } from '../../core/app/schema';
+import { AppCollectionName } from '../../core/app/schema';
 
 const OutLinkSchema = new Schema({
   shareId: {
@@ -25,12 +24,12 @@ const OutLinkSchema = new Schema({
   },
   appId: {
     type: Schema.Types.ObjectId,
-    ref: appCollectionName,
+    ref: AppCollectionName,
     required: true
   },
   type: {
     type: String,
-    default: OutLinkTypeEnum.share
+    required: true
   },
   name: {
     type: String,
@@ -62,6 +61,26 @@ const OutLinkSchema = new Schema({
     hookUrl: {
       type: String
     }
+  },
+  app: {
+    appId: {
+      type: String
+    },
+    appSecret: {
+      type: String
+    },
+    encryptKey: {
+      type: String
+    },
+    verificationToken: {
+      type: String
+    }
+  },
+  immediateResponse: {
+    type: String
+  },
+  defaultResponse: {
+    type: String
   }
 });
 

packages/service/support/permission/auth/app.ts

@@ -30,12 +30,10 @@ export async function authApp({
     // get app
     const app = await MongoApp.findOne({ _id: appId, teamId }).lean();
     if (!app) {
-      return Promise.reject(AppErrEnum.unAuthApp);
+      return Promise.reject(AppErrEnum.unExist);
     }
 
-    const isOwner =
+    const isOwner = String(app.tmbId) === tmbId;
-      role !== TeamMemberRoleEnum.visitor &&
-      (String(app.tmbId) === tmbId || role === TeamMemberRoleEnum.owner);
     const canWrite =
       isOwner ||
       (app.permission === PermissionTypeEnum.public && role !== TeamMemberRoleEnum.visitor);

packages/service/support/user/controller.ts

@@ -22,7 +22,10 @@ export async function getUserDetail({
 }): Promise<UserType> {
   const tmb = await (async () => {
     if (tmbId) {
-      return getTmbInfoByTmbId({ tmbId });
+      try {
+        const result = await getTmbInfoByTmbId({ tmbId });
+        return result;
+      } catch (error) {}
     }
     if (userId) {
       return getUserDefaultTeam({ userId });

packages/service/worker/file/extension/csv.ts

@@ -1,9 +1,9 @@
 import Papa from 'papaparse';
-import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
+import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
 import { readFileRawText } from './rawText';
 
 // 加载源文件内容
-export const readCsvRawText = async (params: ReadFileByBufferParams): Promise<ReadFileResponse> => {
+export const readCsvRawText = async (params: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
   const { rawText } = readFileRawText(params);
 
   const csvArr = Papa.parse(rawText).data as string[][];

packages/service/worker/file/extension/docx.ts

@@ -0,0 +1,23 @@
+import mammoth from 'mammoth';
+import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
+import { html2md } from '../../htmlStr2Md/utils';
+
+/**
+ * read docx to markdown
+ */
+export const readDocsFile = async ({ buffer }: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
+  try {
+    const { value: html } = await mammoth.convertToHtml({
+      buffer
+    });
+
+    const rawText = html2md(html);
+
+    return {
+      rawText
+    };
+  } catch (error) {
+    console.log('error doc read:', error);
+    return Promise.reject('Can not read doc file, please convert to PDF');
+  }
+};

packages/service/worker/file/extension/html.ts

@@ -0,0 +1,13 @@
+import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
+import { readFileRawText } from './rawText';
+import { html2md } from '../../htmlStr2Md/utils';
+
+export const readHtmlRawText = async (params: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
+  const { rawText: html } = readFileRawText(params);
+
+  const rawText = html2md(html);
+
+  return {
+    rawText
+  };
+};

packages/service/worker/file/extension/pdf.ts

@@ -1,7 +1,7 @@
 import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs';
 // @ts-ignore
 import('pdfjs-dist/legacy/build/pdf.worker.min.mjs');
-import { ReadFileByBufferParams, ReadFileResponse } from './type';
+import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
 
 type TokenType = {
   str: string;
@@ -13,9 +13,7 @@ type TokenType = {
   hasEOL: boolean;
 };
 
-export const readPdfFile = async ({
+export const readPdfFile = async ({ buffer }: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
-  buffer
-}: ReadFileByBufferParams): Promise<ReadFileResponse> => {
   const readPDFPage = async (doc: any, pageNo: number) => {
     const page = await doc.getPage(pageNo);
     const tokenizedText = await page.getTextContent();
@@ -65,7 +63,6 @@ export const readPdfFile = async ({
   loadingTask.destroy();
 
   return {
-    rawText: pageTexts.join(''),
+    rawText: pageTexts.join('')
-    metadata: {}
   };
 };

packages/service/worker/file/extension/pptx.ts

@@ -1,11 +1,11 @@
-import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
+import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
 // import { parseOfficeAsync } from 'officeparser';
-import { parseOffice } from './parseOffice';
+import { parseOffice } from '../parseOffice';
 
 export const readPptxRawText = async ({
   buffer,
   encoding
-}: ReadFileByBufferParams): Promise<ReadFileResponse> => {
+}: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
   const result = await parseOffice({
     buffer,
     encoding: encoding as BufferEncoding,

packages/service/worker/file/extension/rawText.ts

@@ -1,5 +1,5 @@
-import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
 import iconv from 'iconv-lite';
+import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
 
 const rawEncodingList = [
   'ascii',
@@ -17,7 +17,7 @@ const rawEncodingList = [
 ];
 
 // 加载源文件内容
-export const readFileRawText = ({ buffer, encoding }: ReadFileByBufferParams): ReadFileResponse => {
+export const readFileRawText = ({ buffer, encoding }: ReadRawTextByBuffer): ReadFileResponse => {
   const content = rawEncodingList.includes(encoding)
     ? buffer.toString(encoding as BufferEncoding)
     : iconv.decode(buffer, 'gbk');

packages/service/worker/file/extension/xlsx.ts

@@ -1,10 +1,10 @@
-import { ReadFileByBufferParams, ReadFileResponse } from './type.d';
+import { ReadRawTextByBuffer, ReadFileResponse } from '../type';
 import xlsx from 'node-xlsx';
 import Papa from 'papaparse';
 
 export const readXlsxRawText = async ({
   buffer
-}: ReadFileByBufferParams): Promise<ReadFileResponse> => {
+}: ReadRawTextByBuffer): Promise<ReadFileResponse> => {
   const result = xlsx.parse(buffer, {
     skipHidden: false,
     defval: ''

packages/service/worker/file/parseOffice.ts

@@ -2,8 +2,8 @@ import { getNanoid } from '@fastgpt/global/common/string/tools';
 import fs from 'fs';
 import decompress from 'decompress';
 import { DOMParser } from '@xmldom/xmldom';
-import { clearDirFiles } from '../utils';
+import { clearDirFiles } from '../../common/file/utils';
-import { addLog } from '../../system/log';
+import { addLog } from '../../common/system/log';
 
 const DEFAULTDECOMPRESSSUBLOCATION = '/tmp';
 
@@ -44,9 +44,13 @@ const parsePowerPoint = async ({
   }
 
   // Returning an array of all the xml contents read using fs.readFileSync
-  const xmlContentArray = files.map((file) =>
+  const xmlContentArray = files.map((file) => {
-    fs.readFileSync(`${decompressPath}/${file.path}`, encoding)
+    try {
-  );
+      return fs.readFileSync(`${decompressPath}/${file.path}`, encoding);
+    } catch (err) {
+      return fs.readFileSync(`${decompressPath}/${file.path}`, 'utf-8');
+    }
+  });
 
   let responseArr: string[] = [];
 
@@ -95,9 +99,15 @@ export const parseOffice = async ({
   //   const decompressPath = `${DEFAULTDECOMPRESSSUBLOCATION}/test`;
 
   // write new file
-  fs.writeFileSync(filepath, buffer, {
+  try {
-    encoding
+    fs.writeFileSync(filepath, buffer, {
-  });
+      encoding
+    });
+  } catch (err) {
+    fs.writeFileSync(filepath, buffer, {
+      encoding: 'utf-8'
+    });
+  }
 
   const text = await (async () => {
     try {

packages/service/worker/file/read.ts

@@ -0,0 +1,71 @@
+import { parentPort } from 'worker_threads';
+import { readFileRawText } from './extension/rawText';
+import { ReadRawTextByBuffer, ReadRawTextProps } from './type';
+import { readHtmlRawText } from './extension/html';
+import { readPdfFile } from './extension/pdf';
+import { readDocsFile } from './extension/docx';
+import { readPptxRawText } from './extension/pptx';
+import { readXlsxRawText } from './extension/xlsx';
+import { readCsvRawText } from './extension/csv';
+
+parentPort?.on('message', async (props: ReadRawTextProps<Uint8Array>) => {
+  const readFileRawContent = async (params: ReadRawTextByBuffer) => {
+    switch (params.extension) {
+      case 'txt':
+      case 'md':
+        return readFileRawText(params);
+      case 'html':
+        return readHtmlRawText(params);
+      case 'pdf':
+        return readPdfFile(params);
+      case 'docx':
+        return readDocsFile(params);
+      case 'pptx':
+        return readPptxRawText(params);
+      case 'xlsx':
+        const xlsxResult = await readXlsxRawText(params);
+        if (params.csvFormat) {
+          return {
+            rawText: xlsxResult.formatText || ''
+          };
+        }
+        return {
+          rawText: xlsxResult.rawText
+        };
+      case 'csv':
+        const csvResult = await readCsvRawText(params);
+        if (params.csvFormat) {
+          return {
+            rawText: csvResult.formatText || ''
+          };
+        }
+        return {
+          rawText: csvResult.rawText
+        };
+      default:
+        return Promise.reject('Only support .txt, .md, .html, .pdf, .docx, pptx, .csv, .xlsx');
+    }
+  };
+
+  //   params.buffer: Uint8Array -> buffer
+  const buffer = Buffer.from(props.buffer);
+  const newProps: ReadRawTextByBuffer = {
+    ...props,
+    buffer
+  };
+
+  try {
+    parentPort?.postMessage({
+      type: 'success',
+      data: await readFileRawContent(newProps)
+    });
+  } catch (error) {
+    console.log(error);
+    parentPort?.postMessage({
+      type: 'error',
+      data: error
+    });
+  }
+
+  global?.close?.();
+});

packages/service/worker/file/type.d.ts

@@ -0,0 +1,15 @@
+import { ReadFileByBufferParams } from '../../common/file/read/type';
+
+export type ReadRawTextProps<T> = {
+  csvFormat?: boolean;
+  extension: string;
+  buffer: T;
+  encoding: string;
+};
+
+export type ReadRawTextByBuffer = ReadRawTextProps<Buffer>;
+
+export type ReadFileResponse = {
+  rawText: string;
+  formatText?: string;
+};

packages/service/worker/htmlStr2Md.ts

@@ -1,60 +0,0 @@
-import { parentPort } from 'worker_threads';
-import TurndownService from 'turndown';
-//@ts-ignore
-import domino from 'domino';
-//@ts-ignore
-import * as turndownPluginGfm from 'joplin-turndown-plugin-gfm';
-
-const turndownService = new TurndownService({
-  headingStyle: 'atx',
-  bulletListMarker: '-',
-  codeBlockStyle: 'fenced',
-  fence: '```',
-  emDelimiter: '_',
-  strongDelimiter: '**',
-  linkStyle: 'inlined',
-  linkReferenceStyle: 'full'
-});
-parentPort?.on('message', (params: { html: string }) => {
-  const html2md = (html: string): string => {
-    try {
-      const window = domino.createWindow(html);
-      const document = window.document;
-
-      turndownService.remove(['i', 'script', 'iframe']);
-      turndownService.addRule('codeBlock', {
-        filter: 'pre',
-        replacement(_, node) {
-          const content = node.textContent?.trim() || '';
-          // @ts-ignore
-          const codeName = node?._attrsByQName?.class?.data?.trim() || '';
-
-          return `\n\`\`\`${codeName}\n${content}\n\`\`\`\n`;
-        }
-      });
-
-      turndownService.use(turndownPluginGfm.gfm);
-
-      // @ts-ignore
-      return turndownService.turndown(document);
-    } catch (error) {
-      return '';
-    }
-  };
-
-  try {
-    const md = html2md(params?.html || '');
-
-    parentPort?.postMessage({
-      type: 'success',
-      data: md
-    });
-  } catch (error) {
-    parentPort?.postMessage({
-      type: 'error',
-      data: error
-    });
-  }
-
-  global?.close?.();
-});

packages/service/worker/htmlStr2Md/index.ts

@@ -0,0 +1,20 @@
+import { parentPort } from 'worker_threads';
+import { html2md } from './utils';
+
+parentPort?.on('message', (params: { html: string }) => {
+  try {
+    const md = html2md(params?.html || '');
+
+    parentPort?.postMessage({
+      type: 'success',
+      data: md
+    });
+  } catch (error) {
+    parentPort?.postMessage({
+      type: 'error',
+      data: error
+    });
+  }
+
+  global?.close?.();
+});