Perf input guide (#1557)

* perf: input guide code * perf: input guide ui * Chat input guide api * Update app chat config store * perf: app chat config field * perf: app context * perf: params * fix: ts * perf: filter private config * perf: filter private config * perf: import workflow * perf: limit max tip amount
2024-05-21 17:52:04 +08:00
parent 8e8ceb7439
commit fb368a581c
123 changed files with 2124 additions and 1805 deletions
--- a/packages/global/core/app/constants.ts
+++ b/packages/global/core/app/constants.ts
@@ -1,4 +1,4 @@
-import { AppWhisperConfigType } from './type';
+import { AppTTSConfigType, AppWhisperConfigType } from './type';

 export enum AppTypeEnum {
  simple = 'simple',
@@ -13,14 +13,16 @@ export const AppTypeMap = {
  }
 };

+export const defaultTTSConfig: AppTTSConfigType = { type: 'web' };
+
 export const defaultWhisperConfig: AppWhisperConfigType = {
  open: false,
  autoSend: false,
  autoTTSResponse: false
 };

-export const defaultQuestionGuideTextConfig = {
+export const defaultChatInputGuideConfig = {
  open: false,
  textList: [],
-  customURL: ''
+  customUrl: ''
 };
--- a/packages/global/core/app/type.d.ts
+++ b/packages/global/core/app/type.d.ts
@@ -8,7 +8,7 @@ import { DatasetSearchModeEnum } from '../dataset/constants';
 import { TeamTagSchema as TeamTagsSchemaType } from '@fastgpt/global/support/user/team/type.d';
 import { StoreEdgeItemType } from '../workflow/type/edge';

-export interface AppSchema {
+export type AppSchema = {
  _id: string;
  teamId: string;
  tmbId: string;
@@ -23,13 +23,14 @@ export interface AppSchema {
  edges: StoreEdgeItemType[];

  // App system config
+  chatConfig: AppChatConfigType;
  scheduledTriggerConfig?: AppScheduledTriggerConfigType | null;
  scheduledTriggerNextTime?: Date;

  permission: `${PermissionTypeEnum}`;
  inited?: boolean;
  teamTags: string[];
-}
+};

 export type AppListItemType = {
  _id: string;
@@ -66,33 +67,19 @@ export type AppSimpleEditFormType = {
    datasetSearchExtensionBg?: string;
  };
  selectedTools: FlowNodeTemplateType[];
-  userGuide: {
-    welcomeText: string;
-    variables: {
-      id: string;
-      key: string;
-      label: string;
-      type: `${VariableInputEnum}`;
-      required: boolean;
-      maxLen: number;
-      enums: {
-        value: string;
-      }[];
-    }[];
-    questionGuide: boolean;
-    tts: {
-      type: 'none' | 'web' | 'model';
-      model?: string | undefined;
-      voice?: string | undefined;
-      speed?: number | undefined;
-    };
-    whisper: AppWhisperConfigType;
-    scheduleTrigger: AppScheduledTriggerConfigType | null;
-    questionGuideText: AppQuestionGuideTextConfigType;
-  };
+  chatConfig: AppChatConfigType;
 };

-/* app function config */
+/* app chat config type */
+export type AppChatConfigType = {
+  welcomeText?: string;
+  variables?: VariableItemType[];
+  questionGuide?: boolean;
+  ttsConfig?: AppTTSConfigType;
+  whisperConfig?: AppWhisperConfigType;
+  scheduledTriggerConfig?: AppScheduledTriggerConfigType;
+  chatInputGuide?: ChatInputGuideConfigType;
+};
 export type SettingAIDataType = {
  model: string;
  temperature: number;
@@ -125,10 +112,9 @@ export type AppWhisperConfigType = {
  autoTTSResponse: boolean;
 };
 // question guide text
-export type AppQuestionGuideTextConfigType = {
+export type ChatInputGuideConfigType = {
  open: boolean;
-  textList: string[];
-  customURL: string;
+  customUrl: string;
 };
 // interval timer
 export type AppScheduledTriggerConfigType = {
--- a/packages/global/core/app/utils.ts
+++ b/packages/global/core/app/utils.ts
@@ -1,50 +1,42 @@
-import type { AppSimpleEditFormType } from '../app/type';
+import type { AppChatConfigType, AppSimpleEditFormType } from '../app/type';
 import { FlowNodeTypeEnum } from '../workflow/node/constant';
 import { NodeInputKeyEnum, FlowNodeTemplateTypeEnum } from '../workflow/constants';
 import type { FlowNodeInputItemType } from '../workflow/type/io.d';
-import { getGuideModule, splitGuideModule } from '../workflow/utils';
+import { getAppChatConfig } from '../workflow/utils';
 import { StoreNodeItemType } from '../workflow/type';
 import { DatasetSearchModeEnum } from '../dataset/constants';
-import { defaultQuestionGuideTextConfig, defaultWhisperConfig } from './constants';

-export const getDefaultAppForm = (): AppSimpleEditFormType => {
-  return {
-    aiSettings: {
-      model: 'gpt-3.5-turbo',
-      systemPrompt: '',
-      temperature: 0,
-      isResponseAnswerText: true,
-      maxHistories: 6,
-      maxToken: 4000
-    },
-    dataset: {
-      datasets: [],
-      similarity: 0.4,
-      limit: 1500,
-      searchMode: DatasetSearchModeEnum.embedding,
-      usingReRank: false,
-      datasetSearchUsingExtensionQuery: true,
-      datasetSearchExtensionBg: ''
-    },
-    selectedTools: [],
-    userGuide: {
-      welcomeText: '',
-      variables: [],
-      questionGuide: false,
-      tts: {
-        type: 'web'
-      },
-      whisper: defaultWhisperConfig,
-      scheduleTrigger: null,
-      questionGuideText: defaultQuestionGuideTextConfig
-    }
-  };
-};
+export const getDefaultAppForm = (): AppSimpleEditFormType => ({
+  aiSettings: {
+    model: 'gpt-3.5-turbo',
+    systemPrompt: '',
+    temperature: 0,
+    isResponseAnswerText: true,
+    maxHistories: 6,
+    maxToken: 4000
+  },
+  dataset: {
+    datasets: [],
+    similarity: 0.4,
+    limit: 1500,
+    searchMode: DatasetSearchModeEnum.embedding,
+    usingReRank: false,
+    datasetSearchUsingExtensionQuery: true,
+    datasetSearchExtensionBg: ''
+  },
+  selectedTools: [],
+  chatConfig: {}
+});

 /* format app nodes to edit form */
-export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => {
+export const appWorkflow2Form = ({
+  nodes,
+  chatConfig
+}: {
+  nodes: StoreNodeItemType[];
+  chatConfig: AppChatConfigType;
+}) => {
  const defaultAppForm = getDefaultAppForm();
-
  const findInputValueByKey = (inputs: FlowNodeInputItemType[], key: string) => {
    return inputs.find((item) => item.key === key)?.value;
  };
@@ -103,26 +95,6 @@ export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => {
        node.inputs,
        NodeInputKeyEnum.datasetSearchExtensionBg
      );
-    } else if (node.flowNodeType === FlowNodeTypeEnum.systemConfig) {
-      const {
-        welcomeText,
-        variableNodes,
-        questionGuide,
-        ttsConfig,
-        whisperConfig,
-        scheduledTriggerConfig,
-        questionGuideText
-      } = splitGuideModule(getGuideModule(nodes));
-
-      defaultAppForm.userGuide = {
-        welcomeText: welcomeText,
-        variables: variableNodes,
-        questionGuide: questionGuide,
-        tts: ttsConfig,
-        whisper: whisperConfig,
-        scheduleTrigger: scheduledTriggerConfig,
-        questionGuideText: questionGuideText
-      };
    } else if (node.flowNodeType === FlowNodeTypeEnum.pluginModule) {
      if (!node.pluginId) return;

@@ -139,6 +111,12 @@ export const appWorkflow2Form = ({ nodes }: { nodes: StoreNodeItemType[] }) => {
        outputs: node.outputs,
        templateType: FlowNodeTemplateTypeEnum.other
      });
+    } else if (node.flowNodeType === FlowNodeTypeEnum.systemConfig) {
+      defaultAppForm.chatConfig = getAppChatConfig({
+        chatConfig,
+        systemConfigNode: node,
+        isPublicFetch: true
+      });
    }
  });

--- a/packages/global/core/app/version.d.ts
+++ b/packages/global/core/app/version.d.ts
@@ -1,5 +1,6 @@
 import { StoreNodeItemType } from '../workflow/type';
 import { StoreEdgeItemType } from '../workflow/type/edge';
+import { AppChatConfigType } from './type';

 export type AppVersionSchemaType = {
  _id: string;
@@ -7,4 +8,5 @@ export type AppVersionSchemaType = {
  time: Date;
  nodes: StoreNodeItemType[];
  edges: StoreEdgeItemType[];
+  chatConfig: AppChatConfigType;
 };
--- a/packages/global/core/chat/inputGuide/type.d.ts
+++ b/packages/global/core/chat/inputGuide/type.d.ts
@@ -0,0 +1,5 @@
+export type ChatInputGuideSchemaType = {
+  _id: string;
+  appId: string;
+  text: string;
+};
--- a/packages/global/core/chat/type.d.ts
+++ b/packages/global/core/chat/type.d.ts
@@ -10,7 +10,7 @@ import {
 import { FlowNodeTypeEnum } from '../workflow/node/constant';
 import { NodeOutputKeyEnum } from '../workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '../workflow/runtime/constants';
-import { AppSchema, VariableItemType } from '../app/type';
+import { AppChatConfigType, AppSchema, VariableItemType } from '../app/type';
 import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
 import { DatasetSearchModeEnum } from '../dataset/constants';
 import { ChatBoxInputType } from '../../../../projects/app/src/components/ChatBox/type';
--- a/packages/global/core/workflow/constants.ts
+++ b/packages/global/core/workflow/constants.ts
@@ -45,7 +45,7 @@ export enum NodeInputKeyEnum {
  whisper = 'whisper',
  variables = 'variables',
  scheduleTrigger = 'scheduleTrigger',
-  questionGuideText = 'questionGuideText',
+  chatInputGuide = 'chatInputGuide',

  // entry
  userChatInput = 'userChatInput',
--- a/packages/global/core/workflow/template/system/globalVariable.ts
+++ b/packages/global/core/workflow/template/system/globalVariable.ts
@@ -1,33 +0,0 @@
-import { FlowNodeTemplateTypeEnum, WorkflowIOValueTypeEnum } from '../../constants';
-import { getHandleConfig } from '../utils';
-import { FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
-import { VariableItemType } from '../../../app/type';
-import { FlowNodeTemplateType } from '../../type';
-
-export const getGlobalVariableNode = ({
-  id,
-  variables
-}: {
-  id: string;
-  variables: VariableItemType[];
-}): FlowNodeTemplateType => {
-  return {
-    id,
-    templateType: FlowNodeTemplateTypeEnum.other,
-    flowNodeType: FlowNodeTypeEnum.systemConfig,
-    sourceHandle: getHandleConfig(true, true, true, true),
-    targetHandle: getHandleConfig(true, true, true, true),
-    avatar: '/imgs/workflow/variable.png',
-    name: '全局变量',
-    intro: '',
-    version: '481',
-    inputs: [],
-    outputs: variables.map((item) => ({
-      id: item.key,
-      key: item.key,
-      valueType: WorkflowIOValueTypeEnum.string,
-      type: FlowNodeOutputTypeEnum.static,
-      label: item.label
-    }))
-  };
-};
--- a/packages/global/core/workflow/template/system/systemConfig.ts
+++ b/packages/global/core/workflow/template/system/systemConfig.ts
@@ -1,10 +1,6 @@
-import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
+import { FlowNodeTypeEnum } from '../../node/constant';
 import { FlowNodeTemplateType } from '../../type/index.d';
-import {
-  WorkflowIOValueTypeEnum,
-  NodeInputKeyEnum,
-  FlowNodeTemplateTypeEnum
-} from '../../constants';
+import { FlowNodeTemplateTypeEnum, WorkflowIOValueTypeEnum } from '../../constants';
 import { getHandleConfig } from '../utils';

 export const SystemConfigNode: FlowNodeTemplateType = {
@@ -19,50 +15,6 @@ export const SystemConfigNode: FlowNodeTemplateType = {
  unique: true,
  forbidDelete: true,
  version: '481',
-  inputs: [
-    {
-      key: NodeInputKeyEnum.welcomeText,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.string,
-      label: 'core.app.Welcome Text'
-    },
-    {
-      key: NodeInputKeyEnum.variables,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.any,
-      label: 'core.module.Variable',
-      value: []
-    },
-    {
-      key: NodeInputKeyEnum.questionGuide,
-      valueType: WorkflowIOValueTypeEnum.boolean,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      label: ''
-    },
-    {
-      key: NodeInputKeyEnum.tts,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.any,
-      label: ''
-    },
-    {
-      key: NodeInputKeyEnum.whisper,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.any,
-      label: ''
-    },
-    {
-      key: NodeInputKeyEnum.scheduleTrigger,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.any,
-      label: ''
-    },
-    {
-      key: NodeInputKeyEnum.questionGuideText,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      valueType: WorkflowIOValueTypeEnum.any,
-      label: ''
-    }
-  ],
+  inputs: [],
  outputs: []
 };
--- a/packages/global/core/workflow/utils.ts
+++ b/packages/global/core/workflow/utils.ts
@@ -12,10 +12,15 @@ import type {
  AppTTSConfigType,
  AppWhisperConfigType,
  AppScheduledTriggerConfigType,
-  AppQuestionGuideTextConfigType
+  ChatInputGuideConfigType,
+  AppChatConfigType
 } from '../app/type';
 import { EditorVariablePickerType } from '../../../web/components/common/Textarea/PromptEditor/type';
-import { defaultWhisperConfig } from '../app/constants';
+import {
+  defaultChatInputGuideConfig,
+  defaultTTSConfig,
+  defaultWhisperConfig
+} from '../app/constants';
 import { IfElseResultEnum } from './template/system/ifElse/constant';

 export const getHandleId = (nodeId: string, type: 'source' | 'target', key: string) => {
@@ -41,70 +46,81 @@ export const splitGuideModule = (guideModules?: StoreNodeItemType) => {
  const welcomeText: string =
    guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.welcomeText)?.value || '';

-  const variableNodes: VariableItemType[] =
+  const variables: VariableItemType[] =
    guideModules?.inputs.find((item) => item.key === NodeInputKeyEnum.variables)?.value || [];

  const questionGuide: boolean =
    !!guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.questionGuide)?.value ||
    false;

-  const ttsConfig: AppTTSConfigType = guideModules?.inputs?.find(
-    (item) => item.key === NodeInputKeyEnum.tts
-  )?.value || { type: 'web' };
+  const ttsConfig: AppTTSConfigType =
+    guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.tts)?.value ||
+    defaultTTSConfig;

  const whisperConfig: AppWhisperConfigType =
    guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.whisper)?.value ||
    defaultWhisperConfig;

-  const scheduledTriggerConfig: AppScheduledTriggerConfigType | null =
-    guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.scheduleTrigger)?.value ??
-    null;
+  const scheduledTriggerConfig: AppScheduledTriggerConfigType = guideModules?.inputs?.find(
+    (item) => item.key === NodeInputKeyEnum.scheduleTrigger
+  )?.value;

-  const questionGuideText: AppQuestionGuideTextConfigType = guideModules?.inputs?.find(
-    (item) => item.key === NodeInputKeyEnum.questionGuideText
-  )?.value || {
-    open: false
-  };
+  const chatInputGuide: ChatInputGuideConfigType =
+    guideModules?.inputs?.find((item) => item.key === NodeInputKeyEnum.chatInputGuide)?.value ||
+    defaultChatInputGuideConfig;

  return {
    welcomeText,
-    variableNodes,
+    variables,
    questionGuide,
    ttsConfig,
    whisperConfig,
    scheduledTriggerConfig,
-    questionGuideText
+    chatInputGuide
  };
 };
-export const replaceAppChatConfig = ({
-  node,
-  variableList,
-  welcomeText
+export const getAppChatConfig = ({
+  chatConfig,
+  systemConfigNode,
+  storeVariables,
+  storeWelcomeText,
+  isPublicFetch = false
 }: {
-  node?: StoreNodeItemType;
-  variableList?: VariableItemType[];
-  welcomeText?: string;
-}): StoreNodeItemType | undefined => {
-  if (!node) return;
-  return {
-    ...node,
-    inputs: node.inputs.map((input) => {
-      if (input.key === NodeInputKeyEnum.variables && variableList) {
-        return {
-          ...input,
-          value: variableList
-        };
-      }
-      if (input.key === NodeInputKeyEnum.welcomeText && welcomeText) {
-        return {
-          ...input,
-          value: welcomeText
-        };
-      }
+  chatConfig?: AppChatConfigType;
+  systemConfigNode?: StoreNodeItemType;
+  storeVariables?: VariableItemType[];
+  storeWelcomeText?: string;
+  isPublicFetch: boolean;
+}): AppChatConfigType => {
+  const {
+    welcomeText,
+    variables,
+    questionGuide,
+    ttsConfig,
+    whisperConfig,
+    scheduledTriggerConfig,
+    chatInputGuide
+  } = splitGuideModule(systemConfigNode);

-      return input;
-    })
+  const config: AppChatConfigType = {
+    questionGuide,
+    ttsConfig,
+    whisperConfig,
+    scheduledTriggerConfig,
+    chatInputGuide,
+    ...chatConfig,
+    variables: storeVariables ?? chatConfig?.variables ?? variables,
+    welcomeText: storeWelcomeText ?? chatConfig?.welcomeText ?? welcomeText
  };
+
+  if (!isPublicFetch) {
+    if (config?.chatInputGuide?.customUrl) {
+      config.chatInputGuide.customUrl = '';
+    }
+    config.scheduledTriggerConfig = undefined;
+  }
+
+  return config;
 };

 export const getOrInitModuleInputValue = (input: FlowNodeInputItemType) => {
--- a/packages/service/core/app/controller.ts
+++ b/packages/service/core/app/controller.ts
@@ -2,7 +2,7 @@ import { AppSchema } from '@fastgpt/global/core/app/type';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 import { getLLMModel } from '../ai/model';
-import { MongoAppVersion } from './versionSchema';
+import { MongoAppVersion } from './version/schema';

 export const beforeUpdateAppFormat = <T extends AppSchema['modules'] | undefined>({
  nodes
@@ -55,11 +55,13 @@ export const getAppLatestVersion = async (appId: string, app?: AppSchema) => {
  if (version) {
    return {
      nodes: version.nodes,
-      edges: version.edges
+      edges: version.edges,
+      chatConfig: version.chatConfig || app?.chatConfig || {}
    };
  }
  return {
    nodes: app?.modules || [],
-    edges: app?.edges || []
+    edges: app?.edges || [],
+    chatConfig: app?.chatConfig || {}
  };
 };
--- a/packages/service/core/app/qGuideSchema.ts
+++ b/packages/service/core/app/qGuideSchema.ts
@@ -1,40 +0,0 @@
-import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
-import { connectionMongo, type Model } from '../../common/mongo';
-const { Schema, model, models } = connectionMongo;
-
-export const AppQGuideCollectionName = 'app_question_guides';
-
-type AppQGuideSchemaType = {
-  _id: string;
-  appId: string;
-  teamId: string;
-  text: string;
-};
-
-const AppQGuideSchema = new Schema({
-  appId: {
-    type: Schema.Types.ObjectId,
-    ref: AppQGuideCollectionName,
-    required: true
-  },
-  teamId: {
-    type: Schema.Types.ObjectId,
-    ref: TeamCollectionName,
-    required: true
-  },
-  text: {
-    type: String,
-    default: ''
-  }
-});
-
-try {
-  AppQGuideSchema.index({ appId: 1 });
-} catch (error) {
-  console.log(error);
-}
-
-export const MongoAppQGuide: Model<AppQGuideSchemaType> =
-  models[AppQGuideCollectionName] || model(AppQGuideCollectionName, AppQGuideSchema);
-
-MongoAppQGuide.syncIndexes();
--- a/packages/service/core/app/schema.ts
+++ b/packages/service/core/app/schema.ts
@@ -10,6 +10,16 @@ import {

 export const AppCollectionName = 'apps';

+export const chatConfigType = {
+  welcomeText: String,
+  variables: Array,
+  questionGuide: Boolean,
+  ttsConfig: Object,
+  whisperConfig: Object,
+  scheduledTriggerConfig: Object,
+  chatInputGuide: Object
+};
+
 const AppSchema = new Schema({
  teamId: {
    type: Schema.Types.ObjectId,
@@ -47,6 +57,16 @@ const AppSchema = new Schema({
    default: () => new Date()
  },

+  // role and auth
+  permission: {
+    type: String,
+    enum: Object.keys(PermissionTypeMap),
+    default: PermissionTypeEnum.private
+  },
+  teamTags: {
+    type: [String]
+  },
+
  // tmp store
  modules: {
    type: Array,
@@ -56,6 +76,10 @@ const AppSchema = new Schema({
    type: Array,
    default: []
  },
+  chatConfig: {
+    type: chatConfigType,
+    default: {}
+  },

  scheduledTriggerConfig: {
    cronString: {
@@ -74,14 +98,6 @@ const AppSchema = new Schema({

  inited: {
    type: Boolean
-  },
-  permission: {
-    type: String,
-    enum: Object.keys(PermissionTypeMap),
-    default: PermissionTypeEnum.private
-  },
-  teamTags: {
-    type: [String]
  }
 });

--- a/packages/service/core/app/version/schema.ts
+++ b/packages/service/core/app/version/schema.ts
@@ -1,6 +1,7 @@
-import { connectionMongo, type Model } from '../../common/mongo';
+import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
+import { chatConfigType } from '../schema';

 export const AppVersionCollectionName = 'app_versions';

@@ -21,6 +22,10 @@ const AppVersionSchema = new Schema({
  edges: {
    type: Array,
    default: []
+  },
+  chatConfig: {
+    type: chatConfigType,
+    default: {}
  }
 });

--- a/packages/service/core/chat/inputGuide/schema.ts
+++ b/packages/service/core/chat/inputGuide/schema.ts
@@ -0,0 +1,29 @@
+import { AppCollectionName } from '../../app/schema';
+import { connectionMongo, type Model } from '../../../common/mongo';
+const { Schema, model, models } = connectionMongo;
+import type { ChatInputGuideSchemaType } from '@fastgpt/global/core/chat/inputGuide/type.d';
+
+export const ChatInputGuideCollectionName = 'chat_input_guides';
+
+const ChatInputGuideSchema = new Schema({
+  appId: {
+    type: Schema.Types.ObjectId,
+    ref: AppCollectionName,
+    required: true
+  },
+  text: {
+    type: String,
+    default: ''
+  }
+});
+
+try {
+  ChatInputGuideSchema.index({ appId: 1, text: 1 }, { unique: true });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoChatInputGuide: Model<ChatInputGuideSchemaType> =
+  models[ChatInputGuideCollectionName] || model(ChatInputGuideCollectionName, ChatInputGuideSchema);
+
+MongoChatInputGuide.syncIndexes();
--- a/packages/service/core/dataset/training/controller.ts
+++ b/packages/service/core/dataset/training/controller.ts
@@ -168,7 +168,8 @@ export async function pushDataListToTrainingQueue({
        indexes: item.indexes
      })),
      {
-        session
+        session,
+        ordered: false
      }
    );
  } catch (error: any) {
--- a/packages/service/core/workflow/dispatch/index.ts
+++ b/packages/service/core/workflow/dispatch/index.ts
@@ -44,6 +44,7 @@ import { RuntimeEdgeItemType } from '@fastgpt/global/core/workflow/type/edge';
 import { getReferenceVariableValue } from '@fastgpt/global/core/workflow/runtime/utils';
 import { dispatchSystemConfig } from './init/systemConfig';
 import { dispatchUpdateVariable } from './tools/runUpdateVar';
+import { addLog } from '../../../common/system/log';

 const callbackMap: Record<FlowNodeTypeEnum, Function> = {
  [FlowNodeTypeEnum.workflowStart]: dispatchWorkflowStart,
@@ -137,7 +138,6 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
    }
    if (nodeDispatchUsages) {
      chatNodeUsages = chatNodeUsages.concat(nodeDispatchUsages);
-      props.maxRunTimes -= nodeDispatchUsages.length;
    }
    if (toolResponses !== undefined) {
      if (Array.isArray(toolResponses) && toolResponses.length === 0) return;
@@ -213,9 +213,11 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
        });

        if (status === 'run') {
+          addLog.info(`[dispatchWorkFlow] nodeRunWithActive: ${node.name}`);
          return nodeRunWithActive(node);
        }
        if (status === 'skip') {
+          addLog.info(`[dispatchWorkFlow] nodeRunWithActive: ${node.name}`);
          return nodeRunWithSkip(node);
        }

@@ -275,6 +277,8 @@ export async function dispatchWorkFlow(data: Props): Promise<DispatchFlowRespons
      });
    }

+    props.maxRunTimes--;
+
    // get node running params
    const params = getNodeRunParams(node);

--- a/packages/service/support/permission/type.d.ts
+++ b/packages/service/support/permission/type.d.ts
@@ -1,4 +1,4 @@
-import { NextApiRequest } from 'next';
+import { ApiRequestProps } from '../../type/next';

 export type ReqHeaderAuthType = {
  cookie?: string;
@@ -9,7 +9,7 @@ export type ReqHeaderAuthType = {
  authorization?: string;
 };
 export type AuthModeType = {
-  req: NextApiRequest;
+  req: ApiRequestProps;
  authToken?: boolean;
  authRoot?: boolean;
  authApiKey?: boolean;
--- a/packages/service/support/user/schema.ts
+++ b/packages/service/support/user/schema.ts
@@ -45,7 +45,7 @@ const UserSchema = new Schema({
  inviterId: {
    // 谁邀请注册的
    type: Schema.Types.ObjectId,
-    ref: 'user'
+    ref: userCollectionName
  },
  promotionRate: {
    type: Number,
--- a/packages/web/common/file/utils.ts
+++ b/packages/web/common/file/utils.ts
@@ -1,4 +1,5 @@
 import { getErrText } from '@fastgpt/global/common/error/utils';
+import Papa from 'papaparse';

 export const loadFile2Buffer = ({ file, onError }: { file: File; onError?: (err: any) => void }) =>
  new Promise<ArrayBuffer>((resolve, reject) => {
@@ -29,3 +30,47 @@ export const loadFile2Buffer = ({ file, onError }: { file: File; onError?: (err:
      reject('The browser does not support file content reading');
    }
  });
+
+export const readFileRawText = ({
+  file,
+  onError
+}: {
+  file: File;
+  onError?: (err: any) => void;
+}) => {
+  return new Promise<string>((resolve, reject) => {
+    try {
+      let reader = new FileReader();
+      reader.onload = async ({ target }) => {
+        if (!target?.result) {
+          onError?.('Load file error');
+          return reject('Load file error');
+        }
+        try {
+          resolve(target.result as string);
+        } catch (err) {
+          console.log(err, 'Load file error');
+          onError?.(err);
+
+          reject(getErrText(err, 'Load file error'));
+        }
+      };
+      reader.onerror = (err) => {
+        console.log(err, 'Load file error');
+        onError?.(err);
+
+        reject(getErrText(err, 'Load file error'));
+      };
+      reader.readAsText(file);
+    } catch (error) {
+      reject('The browser does not support file content reading');
+    }
+  });
+};
+
+export const readCsvRawText = async ({ file }: { file: File }) => {
+  const rawText = await readFileRawText({ file });
+  const csvArr = Papa.parse(rawText).data as string[][];
+
+  return csvArr;
+};
--- a/packages/web/components/common/EmptyTip/index.tsx
+++ b/packages/web/components/common/EmptyTip/index.tsx
@@ -10,7 +10,7 @@ type Props = FlexProps & {
 const EmptyTip = ({ text, ...props }: Props) => {
  const { t } = useTranslation();
  return (
-    <Flex mt={5} flexDirection={'column'} alignItems={'center'} pt={'10vh'} {...props}>
+    <Flex mt={5} flexDirection={'column'} alignItems={'center'} py={'10vh'} {...props}>
      <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
      <Box mt={2} color={'myGray.500'}>
        {text || t('common.empty.Common Tip')}
--- a/packages/web/components/common/Icon/index.tsx
+++ b/packages/web/components/common/Icon/index.tsx
@@ -15,7 +15,7 @@ const MyIcon = ({ name, w = 'auto', h = 'auto', ...props }: { name: IconNameType
      .catch((error) => console.log(error));
  }, [name]);

-  return !!name && !!iconPaths[name] ? (
+  return !!IconComponent ? (
    <Icon
      {...IconComponent}
      w={w}
--- a/packages/web/components/common/MyBox/index.tsx
+++ b/packages/web/components/common/MyBox/index.tsx
@@ -9,7 +9,7 @@ type Props = BoxProps & {

 const MyBox = ({ text, isLoading, children, ...props }: Props, ref: any) => {
  return (
-    <Box ref={ref} position={'relative'} {...props}>
+    <Box ref={ref} position={isLoading ? 'relative' : 'unset'} {...props}>
      {isLoading && <Loading fixed={false} text={text} />}
      {children}
    </Box>
--- a/packages/web/components/common/MyModal/index.tsx
+++ b/packages/web/components/common/MyModal/index.tsx
@@ -11,11 +11,13 @@ import {
  useMediaQuery
 } from '@chakra-ui/react';
 import MyIcon from '../Icon';
+import MyBox from '../MyBox';

 export interface MyModalProps extends ModalContentProps {
  iconSrc?: string;
  title?: any;
  isCentered?: boolean;
+  isLoading?: boolean;
  isOpen: boolean;
  onClose?: () => void;
 }
@@ -27,6 +29,7 @@ const MyModal = ({
  title,
  children,
  isCentered,
+  isLoading,
  w = 'auto',
  maxW = ['90vw', '600px'],
  ...props
@@ -39,6 +42,7 @@ const MyModal = ({
      onClose={() => onClose && onClose()}
      autoFocus={false}
      isCentered={isPc ? isCentered : true}
+      blockScrollOnMount={false}
    >
      <ModalOverlay />
      <ModalContent
@@ -78,14 +82,15 @@ const MyModal = ({
          </ModalHeader>
        )}

-        <Box
+        <MyBox
+          isLoading={isLoading}
          overflow={props.overflow || 'overlay'}
          h={'100%'}
          display={'flex'}
          flexDirection={'column'}
        >
          {children}
-        </Box>
+        </MyBox>
      </ModalContent>
    </Modal>
  );
--- a/packages/web/components/common/String/HighlightText.tsx
+++ b/packages/web/components/common/String/HighlightText.tsx
@@ -0,0 +1,40 @@
+import { Box } from '@chakra-ui/react';
+import React from 'react';
+
+const HighlightText = ({
+  rawText,
+  matchText,
+  color = 'primary.600'
+}: {
+  rawText: string;
+  matchText: string;
+  color?: string;
+}) => {
+  const regex = new RegExp(`(${matchText})`, 'gi');
+  const parts = rawText.split(regex);
+
+  return (
+    <Box>
+      {parts.map((part, index) => {
+        let highLight = part.toLowerCase() === matchText.toLowerCase();
+
+        if (highLight) {
+          parts.find((item, i) => {
+            if (i >= index) return;
+            if (item.toLowerCase() === matchText.toLowerCase()) {
+              highLight = false;
+            }
+          });
+        }
+
+        return (
+          <Box as="span" key={index} color={highLight ? color : 'inherit'}>
+            {part}
+          </Box>
+        );
+      })}
+    </Box>
+  );
+};
+
+export default HighlightText;
--- a/packages/web/hooks/useRequest.tsx
+++ b/packages/web/hooks/useRequest.tsx
@@ -3,6 +3,7 @@ import { useMutation } from '@tanstack/react-query';
 import type { UseMutationOptions } from '@tanstack/react-query';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useTranslation } from 'next-i18next';
+import { useRequest as ahooksUseRequest } from 'ahooks';

 interface Props extends UseMutationOptions<any, any, any, any> {
  successToast?: string | null;
@@ -39,3 +40,50 @@ export const useRequest = ({ successToast, errorToast, onSuccess, onError, ...pr

  return mutation;
 };
+
+type UseRequestFunProps<TData, TParams extends any[]> = Parameters<
+  typeof ahooksUseRequest<TData, TParams>
+>;
+export const useRequest2 = <TData, TParams extends any[]>(
+  server: UseRequestFunProps<TData, TParams>[0],
+  options: UseRequestFunProps<TData, TParams>[1] & {
+    errorToast?: string;
+    successToast?: string;
+  } = {},
+  plugin?: UseRequestFunProps<TData, TParams>[2]
+) => {
+  const { t } = useTranslation();
+  const { errorToast, successToast, ...rest } = options || {};
+  const { toast } = useToast();
+
+  const res = ahooksUseRequest<TData, TParams>(
+    server,
+    {
+      ...rest,
+      onError: (err, params) => {
+        rest?.onError?.(err, params);
+        if (errorToast !== undefined) {
+          const errText = t(getErrText(err, errorToast || ''));
+          if (errText) {
+            toast({
+              title: errText,
+              status: 'error'
+            });
+          }
+        }
+      },
+      onSuccess: (res, params) => {
+        rest?.onSuccess?.(res, params);
+        if (successToast) {
+          toast({
+            title: successToast,
+            status: 'success'
+          });
+        }
+      }
+    },
+    plugin
+  );
+
+  return res;
+};
--- a/packages/web/hooks/useScrollPagination.tsx
+++ b/packages/web/hooks/useScrollPagination.tsx
@@ -1,9 +1,17 @@
-import { useRef, useState, useEffect } from 'react';
+import React, { useRef, useState, useEffect } from 'react';
 import { Box, BoxProps } from '@chakra-ui/react';
 import { useToast } from './useToast';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { PaginationProps, PaginationResponse } from '../common/fetch/type';
-import { useBoolean, useLockFn, useMemoizedFn, useMount, useScroll, useVirtualList } from 'ahooks';
+import {
+  useBoolean,
+  useLockFn,
+  useMemoizedFn,
+  useMount,
+  useScroll,
+  useVirtualList,
+  useRequest
+} from 'ahooks';
 import MyBox from '../components/common/MyBox';
 import { useTranslation } from 'next-i18next';

@@ -13,12 +21,19 @@ export function useScrollPagination<
 >(
  api: (data: TParams) => Promise<TData>,
  {
+    debounceWait,
+    throttleWait,
+    refreshDeps,
    itemHeight = 50,
    overscan = 10,

    pageSize = 10,
    defaultParams = {}
  }: {
+    debounceWait?: number;
+    throttleWait?: number;
+    refreshDeps?: any[];
+
    itemHeight: number;
    overscan?: number;

@@ -45,7 +60,7 @@ export function useScrollPagination<
  });

  const loadData = useLockFn(async (num: number = current) => {
-    if (noMore.current) return;
+    if (noMore.current && num !== 1) return;

    setTrue();

@@ -59,7 +74,7 @@ export function useScrollPagination<
      setCurrent(num);

      if (num === 1) {
-        // reload
+        // init or reload
        setData(res.list);
        noMore.current = res.list.length >= res.total;
      } else {
@@ -78,34 +93,48 @@ export function useScrollPagination<
    setFalse();
  });

+  const scroll2Top = () => {
+    if (containerRef.current) {
+      containerRef.current.scrollTop = 0;
+    }
+  };
+
  const ScrollList = useMemoizedFn(
    ({
      children,
+      EmptyChildren,
      isLoading,
      ...props
-    }: { children: React.ReactNode; isLoading?: boolean } & BoxProps) => {
+    }: {
+      children: React.ReactNode;
+      EmptyChildren?: React.ReactNode;
+      isLoading?: boolean;
+    } & BoxProps) => {
      return (
        <>
          <MyBox isLoading={isLoading} ref={containerRef} overflow={'overlay'} {...props}>
            <Box ref={wrapperRef}>{children}</Box>
+            {noMore.current && list.length > 0 && (
+              <Box py={4} textAlign={'center'} color={'myGray.600'} fontSize={'sm'}>
+                {t('common.No more data')}
+              </Box>
+            )}
+            {list.length === 0 && !isLoading && EmptyChildren && <>{EmptyChildren}</>}
          </MyBox>
-          {noMore.current && (
-            <Box pb={2} textAlign={'center'} color={'myGray.600'} fontSize={'sm'}>
-              {t('common.No more data')}
-            </Box>
-          )}
        </>
      );
    }
  );

-  useMount(() => {
-    loadData(1);
+  useRequest(() => loadData(1), {
+    refreshDeps,
+    debounceWait: data.length === 0 ? 0 : debounceWait,
+    throttleWait
  });

  const scroll = useScroll(containerRef);
  useEffect(() => {
-    if (!containerRef.current) return;
+    if (!containerRef.current || list.length === 0) return;

    const { scrollTop, scrollHeight, clientHeight } = containerRef.current;

@@ -118,8 +147,10 @@ export function useScrollPagination<
    containerRef,
    list,
    data,
+    setData,
    isLoading,
    ScrollList,
-    fetchData: loadData
+    fetchData: loadData,
+    scroll2Top
  };
 }
--- a/packages/web/package.json
+++ b/packages/web/package.json
@@ -26,7 +26,6 @@
    "lodash": "^4.17.21",
    "next-i18next": "15.2.0",
    "papaparse": "^5.4.1",
-    "pdfjs-dist": "4.0.269",
    "react": "18.3.1",
    "use-context-selector": "^1.4.4",
    "react-day-picker": "^8.7.1",