monorepo packages (#344)

projects/app/src/utils/adapt.ts

@@ -0,0 +1,155 @@
+import { formatPrice } from '@fastgpt/common/bill/index';
+import type { BillSchema } from '@/types/common/bill';
+import type { UserBillType } from '@/types/user';
+import { ChatItemType } from '@/types/chat';
+import { ChatCompletionRequestMessageRoleEnum } from 'openai';
+import { ChatRoleEnum } from '@/constants/chat';
+import type { MessageItemType } from '@/pages/api/openapi/v1/chat/completions';
+import type { AppModuleItemType } from '@/types/app';
+import type { FlowModuleItemType } from '@/types/flow';
+import type { Edge, Node } from 'reactflow';
+import { connectionLineStyle } from '@/constants/flow';
+import { customAlphabet } from 'nanoid';
+import { EmptyModule, ModuleTemplatesFlat } from '@/constants/flow/ModuleTemplate';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 6);
+
+export const adaptBill = (bill: BillSchema): UserBillType => {
+  return {
+    id: bill._id,
+    source: bill.source,
+    time: bill.time,
+    total: formatPrice(bill.total),
+    appName: bill.appName,
+    list: bill.list
+  };
+};
+
+export const gptMessage2ChatType = (messages: MessageItemType[]): ChatItemType[] => {
+  const roleMap: Record<`${ChatCompletionRequestMessageRoleEnum}`, `${ChatRoleEnum}`> = {
+    [ChatCompletionRequestMessageRoleEnum.Assistant]: ChatRoleEnum.AI,
+    [ChatCompletionRequestMessageRoleEnum.User]: ChatRoleEnum.Human,
+    [ChatCompletionRequestMessageRoleEnum.System]: ChatRoleEnum.System,
+    [ChatCompletionRequestMessageRoleEnum.Function]: ChatRoleEnum.Human
+  };
+
+  return messages.map((item) => ({
+    dataId: item.dataId,
+    obj: roleMap[item.role],
+    value: item.content || ''
+  }));
+};
+
+export const textAdaptGptResponse = ({
+  text,
+  model = '',
+  finish_reason = null,
+  extraData = {}
+}: {
+  model?: string;
+  text: string | null;
+  finish_reason?: null | 'stop';
+  extraData?: Object;
+}) => {
+  return JSON.stringify({
+    ...extraData,
+    id: '',
+    object: '',
+    created: 0,
+    model,
+    choices: [{ delta: text === null ? {} : { content: text }, index: 0, finish_reason }]
+  });
+};
+
+export const appModule2FlowNode = ({
+  item,
+  onChangeNode,
+  onDelNode,
+  onDelEdge,
+  onCopyNode,
+  onCollectionNode
+}: {
+  item: AppModuleItemType;
+  onChangeNode: FlowModuleItemType['onChangeNode'];
+  onDelNode: FlowModuleItemType['onDelNode'];
+  onDelEdge: FlowModuleItemType['onDelEdge'];
+  onCopyNode: FlowModuleItemType['onCopyNode'];
+  onCollectionNode: FlowModuleItemType['onCollectionNode'];
+}): Node<FlowModuleItemType> => {
+  // init some static data
+  const template =
+    ModuleTemplatesFlat.find((template) => template.flowType === item.flowType) || EmptyModule;
+
+  const concatInputs = template.inputs.concat(
+    item.inputs.filter(
+      (input) => input.label && !template.inputs.find((item) => item.key === input.key)
+    )
+  );
+  const concatOutputs = item.outputs.concat(
+    template.outputs.filter(
+      (templateOutput) => !item.outputs.find((item) => item.key === templateOutput.key)
+    )
+  );
+
+  // replace item data
+  const moduleItem: FlowModuleItemType = {
+    ...template,
+    ...item,
+    inputs: concatInputs.map((templateInput) => {
+      // use latest inputs
+      const itemInput = item.inputs.find((item) => item.key === templateInput.key) || templateInput;
+      return {
+        ...templateInput,
+        value: itemInput.value
+      };
+    }),
+    outputs: concatOutputs.map((output) => {
+      // unChange outputs
+      const templateOutput = template.outputs.find((item) => item.key === output.key);
+
+      return {
+        ...(templateOutput ? templateOutput : output),
+        targets: output.targets || []
+      };
+    }),
+    onChangeNode,
+    onDelNode,
+    onDelEdge,
+    onCopyNode,
+    onCollectionNode
+  };
+
+  return {
+    id: item.moduleId,
+    type: item.flowType,
+    data: moduleItem,
+    position: item.position || { x: 0, y: 0 }
+  };
+};
+export const appModule2FlowEdge = ({
+  modules,
+  onDelete
+}: {
+  modules: AppModuleItemType[];
+  onDelete: (id: string) => void;
+}) => {
+  const edges: Edge[] = [];
+  modules.forEach((module) =>
+    module.outputs.forEach((output) =>
+      output.targets.forEach((target) => {
+        edges.push({
+          style: connectionLineStyle,
+          source: module.moduleId,
+          target: target.moduleId,
+          sourceHandle: output.key,
+          targetHandle: target.key,
+          id: nanoid(),
+          animated: true,
+          type: 'buttonedge',
+          data: { onDelete }
+        });
+      })
+    )
+  );
+
+  return edges;
+};

projects/app/src/utils/app.ts

@@ -0,0 +1,581 @@
+import type { AppModuleItemType, VariableItemType } from '@/types/app';
+import { chatModelList } from '@/store/static';
+import {
+  FlowInputItemTypeEnum,
+  FlowModuleTypeEnum,
+  FlowValueTypeEnum,
+  SpecialInputKeyEnum
+} from '@/constants/flow';
+import { SystemInputEnum } from '@/constants/app';
+import type { SelectedDatasetType } from '@/types/core/dataset';
+import { FlowInputItemType } from '@/types/flow';
+import type { AIChatProps } from '@/types/core/aiChat';
+
+export type EditFormType = {
+  chatModel: AIChatProps;
+  kb: {
+    list: SelectedDatasetType;
+    searchSimilarity: number;
+    searchLimit: number;
+    searchEmptyText: string;
+  };
+  guide: {
+    welcome: {
+      text: string;
+    };
+  };
+  variables: VariableItemType[];
+};
+export const getDefaultAppForm = (): EditFormType => {
+  const defaultChatModel = chatModelList[0];
+
+  return {
+    chatModel: {
+      model: defaultChatModel?.model,
+      systemPrompt: '',
+      temperature: 0,
+      quotePrompt: '',
+      quoteTemplate: '',
+      maxToken: defaultChatModel ? defaultChatModel.contextMaxToken / 2 : 4000,
+      frequency: 0.5,
+      presence: -0.5
+    },
+    kb: {
+      list: [],
+      searchSimilarity: 0.4,
+      searchLimit: 5,
+      searchEmptyText: ''
+    },
+    guide: {
+      welcome: {
+        text: ''
+      }
+    },
+    variables: []
+  };
+};
+
+export const appModules2Form = (modules: AppModuleItemType[]) => {
+  const defaultAppForm = getDefaultAppForm();
+  const updateVal = ({
+    formKey,
+    inputs,
+    key
+  }: {
+    formKey: string;
+    inputs: FlowInputItemType[];
+    key: string;
+  }) => {
+    const propertyPath = formKey.split('.');
+    let currentObj: any = defaultAppForm;
+    for (let i = 0; i < propertyPath.length - 1; i++) {
+      currentObj = currentObj[propertyPath[i]];
+    }
+
+    const val =
+      inputs.find((item) => item.key === key)?.value ||
+      currentObj[propertyPath[propertyPath.length - 1]];
+
+    currentObj[propertyPath[propertyPath.length - 1]] = val;
+  };
+
+  modules.forEach((module) => {
+    if (module.flowType === FlowModuleTypeEnum.chatNode) {
+      updateVal({
+        formKey: 'chatModel.model',
+        inputs: module.inputs,
+        key: 'model'
+      });
+      updateVal({
+        formKey: 'chatModel.temperature',
+        inputs: module.inputs,
+        key: 'temperature'
+      });
+      updateVal({
+        formKey: 'chatModel.maxToken',
+        inputs: module.inputs,
+        key: 'maxToken'
+      });
+      updateVal({
+        formKey: 'chatModel.systemPrompt',
+        inputs: module.inputs,
+        key: 'systemPrompt'
+      });
+      updateVal({
+        formKey: 'chatModel.quoteTemplate',
+        inputs: module.inputs,
+        key: 'quoteTemplate'
+      });
+      updateVal({
+        formKey: 'chatModel.quotePrompt',
+        inputs: module.inputs,
+        key: 'quotePrompt'
+      });
+    } else if (module.flowType === FlowModuleTypeEnum.kbSearchNode) {
+      updateVal({
+        formKey: 'kb.list',
+        inputs: module.inputs,
+        key: 'kbList'
+      });
+      updateVal({
+        formKey: 'kb.searchSimilarity',
+        inputs: module.inputs,
+        key: 'similarity'
+      });
+      updateVal({
+        formKey: 'kb.searchLimit',
+        inputs: module.inputs,
+        key: 'limit'
+      });
+      // empty text
+      const emptyOutputs = module.outputs.find((item) => item.key === 'isEmpty')?.targets || [];
+      const emptyOutput = emptyOutputs[0];
+      if (emptyOutput) {
+        const target = modules.find((item) => item.moduleId === emptyOutput.moduleId);
+        defaultAppForm.kb.searchEmptyText =
+          target?.inputs?.find((item) => item.key === SpecialInputKeyEnum.answerText)?.value || '';
+      }
+    } else if (module.flowType === FlowModuleTypeEnum.userGuide) {
+      const val =
+        module.inputs.find((item) => item.key === SystemInputEnum.welcomeText)?.value || '';
+      if (val) {
+        defaultAppForm.guide.welcome = {
+          text: val
+        };
+      }
+    } else if (module.flowType === FlowModuleTypeEnum.variable) {
+      defaultAppForm.variables =
+        module.inputs.find((item) => item.key === SystemInputEnum.variables)?.value || [];
+    }
+  });
+
+  return defaultAppForm;
+};
+
+const chatModelInput = (formData: EditFormType): FlowInputItemType[] => [
+  {
+    key: 'model',
+    value: formData.chatModel.model,
+    type: 'custom',
+    label: '对话模型',
+    connected: true
+  },
+  {
+    key: 'temperature',
+    value: formData.chatModel.temperature,
+    type: 'slider',
+    label: '温度',
+    connected: true
+  },
+  {
+    key: 'maxToken',
+    value: formData.chatModel.maxToken,
+    type: 'custom',
+    label: '回复上限',
+    connected: true
+  },
+  {
+    key: 'systemPrompt',
+    value: formData.chatModel.systemPrompt || '',
+    type: 'textarea',
+    label: '系统提示词',
+    connected: true
+  },
+  {
+    key: 'quoteTemplate',
+    value: formData.chatModel.quoteTemplate || '',
+    type: 'hidden',
+    label: '引用内容模板',
+    connected: true
+  },
+  {
+    key: 'quotePrompt',
+    value: formData.chatModel.quotePrompt || '',
+    type: 'hidden',
+    label: '引用内容提示词',
+    connected: true
+  },
+  {
+    key: 'switch',
+    type: 'target',
+    label: '触发器',
+    connected: formData.kb.list.length > 0
+  },
+  {
+    key: 'quoteQA',
+    type: 'target',
+    label: '引用内容',
+    connected: formData.kb.list.length > 0
+  },
+  {
+    key: 'history',
+    type: 'target',
+    label: '聊天记录',
+    connected: true
+  },
+  {
+    key: 'userChatInput',
+    type: 'target',
+    label: '用户问题',
+    connected: true
+  }
+];
+const welcomeTemplate = (formData: EditFormType): AppModuleItemType[] =>
+  formData.guide?.welcome?.text
+    ? [
+        {
+          name: '用户引导',
+          flowType: FlowModuleTypeEnum.userGuide,
+          inputs: [
+            {
+              key: 'welcomeText',
+              type: 'input',
+              label: '开场白',
+              value: formData.guide.welcome.text,
+              connected: true
+            }
+          ],
+          outputs: [],
+          position: {
+            x: 447.98520778293346,
+            y: 721.4016845336229
+          },
+          moduleId: 'userGuide'
+        }
+      ]
+    : [];
+const variableTemplate = (formData: EditFormType): AppModuleItemType[] =>
+  formData.variables.length > 0
+    ? [
+        {
+          name: '全局变量',
+          flowType: FlowModuleTypeEnum.variable,
+          inputs: [
+            {
+              key: 'variables',
+              value: formData.variables,
+              type: 'systemInput',
+              label: '变量输入',
+              connected: true
+            }
+          ],
+          outputs: [],
+          position: {
+            x: 444.0369195277651,
+            y: 1008.5185781784537
+          },
+          moduleId: 'variable'
+        }
+      ]
+    : [];
+const simpleChatTemplate = (formData: EditFormType): AppModuleItemType[] => [
+  {
+    name: '用户问题(对话入口)',
+    flowType: FlowModuleTypeEnum.questionInput,
+    inputs: [
+      {
+        key: 'userChatInput',
+        connected: true,
+        label: '用户问题',
+        type: 'target'
+      }
+    ],
+    outputs: [
+      {
+        key: 'userChatInput',
+        targets: [
+          {
+            moduleId: 'chatModule',
+            key: 'userChatInput'
+          }
+        ]
+      }
+    ],
+    position: {
+      x: 464.32198615344566,
+      y: 1602.2698463081606
+    },
+    moduleId: 'userChatInput'
+  },
+  {
+    name: '聊天记录',
+    flowType: FlowModuleTypeEnum.historyNode,
+    inputs: [
+      {
+        key: 'maxContext',
+        value: 6,
+        connected: true,
+        type: 'numberInput',
+        label: '最长记录数'
+      },
+      {
+        key: 'history',
+        type: 'hidden',
+        label: '聊天记录',
+        connected: true
+      }
+    ],
+    outputs: [
+      {
+        key: 'history',
+        targets: [
+          {
+            moduleId: 'chatModule',
+            key: 'history'
+          }
+        ]
+      }
+    ],
+    position: {
+      x: 452.5466249541586,
+      y: 1276.3930310334215
+    },
+    moduleId: 'history'
+  },
+  {
+    name: 'AI 对话',
+    flowType: FlowModuleTypeEnum.chatNode,
+    inputs: chatModelInput(formData),
+    showStatus: true,
+    outputs: [
+      {
+        key: 'answerText',
+        label: '模型回复',
+        description: '直接响应，无需配置',
+        type: 'hidden',
+        targets: []
+      },
+      {
+        key: 'finish',
+        label: '回复结束',
+        description: 'AI 回复完成后触发',
+        valueType: 'boolean',
+        type: 'source',
+        targets: []
+      }
+    ],
+    position: {
+      x: 981.9682828103937,
+      y: 890.014595014464
+    },
+    moduleId: 'chatModule'
+  }
+];
+const kbTemplate = (formData: EditFormType): AppModuleItemType[] => [
+  {
+    name: '用户问题(对话入口)',
+    flowType: FlowModuleTypeEnum.questionInput,
+    inputs: [
+      {
+        key: 'userChatInput',
+        label: '用户问题',
+        type: 'target',
+        connected: true
+      }
+    ],
+    outputs: [
+      {
+        key: 'userChatInput',
+        targets: [
+          {
+            moduleId: 'chatModule',
+            key: 'userChatInput'
+          },
+          {
+            moduleId: 'kbSearch',
+            key: 'userChatInput'
+          }
+        ]
+      }
+    ],
+    position: {
+      x: 464.32198615344566,
+      y: 1602.2698463081606
+    },
+    moduleId: 'userChatInput'
+  },
+  {
+    name: '聊天记录',
+    flowType: FlowModuleTypeEnum.historyNode,
+    inputs: [
+      {
+        key: 'maxContext',
+        value: 6,
+        connected: true,
+        type: 'numberInput',
+        label: '最长记录数'
+      },
+      {
+        key: 'history',
+        type: 'hidden',
+        label: '聊天记录',
+        connected: true
+      }
+    ],
+    outputs: [
+      {
+        key: 'history',
+        targets: [
+          {
+            moduleId: 'chatModule',
+            key: 'history'
+          }
+        ]
+      }
+    ],
+    position: {
+      x: 452.5466249541586,
+      y: 1276.3930310334215
+    },
+    moduleId: 'history'
+  },
+  {
+    name: '知识库搜索',
+    flowType: FlowModuleTypeEnum.kbSearchNode,
+    showStatus: true,
+    inputs: [
+      {
+        key: 'kbList',
+        value: formData.kb.list,
+        type: FlowInputItemTypeEnum.custom,
+        label: '关联的知识库',
+        connected: true
+      },
+      {
+        key: 'similarity',
+        value: formData.kb.searchSimilarity,
+        type: FlowInputItemTypeEnum.slider,
+        label: '相似度',
+        connected: true
+      },
+      {
+        key: 'limit',
+        value: formData.kb.searchLimit,
+        type: FlowInputItemTypeEnum.slider,
+        label: '单次搜索上限',
+        connected: true
+      },
+      {
+        key: 'switch',
+        type: FlowInputItemTypeEnum.target,
+        label: '触发器',
+        connected: false
+      },
+      {
+        key: 'userChatInput',
+        type: FlowInputItemTypeEnum.target,
+        label: '用户问题',
+        connected: true
+      }
+    ],
+    outputs: [
+      {
+        key: 'isEmpty',
+        targets: formData.kb.searchEmptyText
+          ? [
+              {
+                moduleId: 'emptyText',
+                key: 'switch'
+              }
+            ]
+          : [
+              {
+                moduleId: 'chatModule',
+                key: 'switch'
+              }
+            ]
+      },
+      {
+        key: 'unEmpty',
+        targets: [
+          {
+            moduleId: 'chatModule',
+            key: 'switch'
+          }
+        ]
+      },
+      {
+        key: 'quoteQA',
+        targets: [
+          {
+            moduleId: 'chatModule',
+            key: 'quoteQA'
+          }
+        ]
+      }
+    ],
+    position: {
+      x: 956.0838440206068,
+      y: 887.462827870246
+    },
+    moduleId: 'kbSearch'
+  },
+  ...(formData.kb.searchEmptyText
+    ? [
+        {
+          name: '指定回复',
+          flowType: FlowModuleTypeEnum.answerNode,
+          inputs: [
+            {
+              key: 'switch',
+              type: FlowInputItemTypeEnum.target,
+              label: '触发器',
+              connected: true
+            },
+            {
+              key: SpecialInputKeyEnum.answerText,
+              value: formData.kb.searchEmptyText,
+              type: FlowInputItemTypeEnum.textarea,
+              valueType: FlowValueTypeEnum.string,
+              label: '回复的内容',
+              connected: true
+            }
+          ],
+          outputs: [],
+          position: {
+            x: 1553.5815811529146,
+            y: 637.8753731306779
+          },
+          moduleId: 'emptyText'
+        }
+      ]
+    : []),
+  {
+    name: 'AI 对话',
+    flowType: FlowModuleTypeEnum.chatNode,
+    inputs: chatModelInput(formData),
+    showStatus: true,
+    outputs: [
+      {
+        key: 'answerText',
+        label: '模型回复',
+        description: '直接响应，无需配置',
+        type: 'hidden',
+        targets: []
+      },
+      {
+        key: 'finish',
+        label: '回复结束',
+        description: 'AI 回复完成后触发',
+        valueType: 'boolean',
+        type: 'source',
+        targets: []
+      }
+    ],
+    position: {
+      x: 1551.71405495818,
+      y: 977.4911578918461
+    },
+    moduleId: 'chatModule'
+  }
+];
+
+export const appForm2Modules = (formData: EditFormType) => {
+  const modules = [
+    ...welcomeTemplate(formData),
+    ...variableTemplate(formData),
+    ...(formData.kb.list.length > 0 ? kbTemplate(formData) : simpleChatTemplate(formData))
+  ];
+
+  return modules as AppModuleItemType[];
+};

projects/app/src/utils/common/adapt/message.ts

@@ -0,0 +1,37 @@
+import type { ChatItemType } from '@/types/chat';
+import { ChatRoleEnum } from '@/constants/chat';
+import { ChatCompletionRequestMessageRoleEnum } from 'openai';
+import type { MessageItemType } from '@/pages/api/openapi/v1/chat/completions';
+
+const chat2Message = {
+  [ChatRoleEnum.AI]: ChatCompletionRequestMessageRoleEnum.Assistant,
+  [ChatRoleEnum.Human]: ChatCompletionRequestMessageRoleEnum.User,
+  [ChatRoleEnum.System]: ChatCompletionRequestMessageRoleEnum.System
+};
+const message2Chat = {
+  [ChatCompletionRequestMessageRoleEnum.System]: ChatRoleEnum.System,
+  [ChatCompletionRequestMessageRoleEnum.User]: ChatRoleEnum.Human,
+  [ChatCompletionRequestMessageRoleEnum.Assistant]: ChatRoleEnum.AI,
+  [ChatCompletionRequestMessageRoleEnum.Function]: 'function'
+};
+
+export function adaptRole_Chat2Message(role: `${ChatRoleEnum}`) {
+  return chat2Message[role];
+}
+export function adaptRole_Message2Chat(role: `${ChatCompletionRequestMessageRoleEnum}`) {
+  return message2Chat[role];
+}
+
+export const adaptChat2GptMessages = ({
+  messages,
+  reserveId
+}: {
+  messages: ChatItemType[];
+  reserveId: boolean;
+}): MessageItemType[] => {
+  return messages.map((item) => ({
+    ...(reserveId && { dataId: item.dataId }),
+    role: chat2Message[item.obj] || ChatCompletionRequestMessageRoleEnum.System,
+    content: item.value || ''
+  }));
+};

projects/app/src/utils/common/tiktoken/index.ts

@@ -0,0 +1,92 @@
+/* Only the token of gpt-3.5-turbo is used */
+import { ChatItemType } from '@/types/chat';
+import { Tiktoken } from 'js-tiktoken/lite';
+import { adaptChat2GptMessages } from '../adapt/message';
+import { ChatCompletionRequestMessageRoleEnum } from 'openai';
+import encodingJson from './cl100k_base.json';
+
+/* init tikToken obj */
+export function getTikTokenEnc() {
+  if (typeof window !== 'undefined' && window.TikToken) {
+    return window.TikToken;
+  }
+  if (typeof global !== 'undefined' && global.TikToken) {
+    return global.TikToken;
+  }
+
+  const enc = new Tiktoken(encodingJson);
+
+  if (typeof window !== 'undefined') {
+    window.TikToken = enc;
+  }
+  if (typeof global !== 'undefined') {
+    global.TikToken = enc;
+  }
+
+  return enc;
+}
+
+/* count one prompt tokens */
+export function countPromptTokens(prompt = '', role: `${ChatCompletionRequestMessageRoleEnum}`) {
+  const enc = getTikTokenEnc();
+  const text = `${role}\n${prompt}`;
+  try {
+    const encodeText = enc.encode(text);
+    return encodeText.length + 3; // 补充 role 估算值
+  } catch (error) {
+    return text.length;
+  }
+}
+
+/* count messages tokens */
+export function countMessagesTokens({ messages }: { messages: ChatItemType[] }) {
+  const adaptMessages = adaptChat2GptMessages({ messages, reserveId: true });
+
+  let totalTokens = 0;
+  for (let i = 0; i < adaptMessages.length; i++) {
+    const item = adaptMessages[i];
+    const tokens = countPromptTokens(item.content, item.role);
+    totalTokens += tokens;
+  }
+
+  return totalTokens;
+}
+
+export function sliceTextByTokens({ text, length }: { text: string; length: number }) {
+  const enc = getTikTokenEnc();
+
+  try {
+    const encodeText = enc.encode(text);
+    return enc.decode(encodeText.slice(0, length));
+  } catch (error) {
+    return text.slice(0, length);
+  }
+}
+
+/* slice messages from top to bottom by maxTokens */
+export function sliceMessagesTB({
+  messages,
+  maxTokens
+}: {
+  messages: ChatItemType[];
+  maxTokens: number;
+}) {
+  const adaptMessages = adaptChat2GptMessages({ messages, reserveId: true });
+  let reduceTokens = maxTokens;
+  let result: ChatItemType[] = [];
+
+  for (let i = 0; i < adaptMessages.length; i++) {
+    const item = adaptMessages[i];
+
+    const tokens = countPromptTokens(item.content, item.role);
+    reduceTokens -= tokens;
+
+    if (reduceTokens > 0) {
+      result.push(messages[i]);
+    } else {
+      break;
+    }
+  }
+
+  return result.length === 0 && messages[0] ? [messages[0]] : result;
+}

projects/app/src/utils/common/tools/text.ts

@@ -0,0 +1,12 @@
+/* 
+    replace {{variable}} to value
+*/
+export function replaceVariable(text: string, obj: Record<string, string>) {
+  for (const key in obj) {
+    const val = obj[key];
+    if (typeof val !== 'string') continue;
+
+    text = text.replace(new RegExp(`{{(${key})}}`, 'g'), val);
+  }
+  return text || '';
+}

projects/app/src/utils/file.ts

@@ -0,0 +1,56 @@
+import { getErrText } from './tools';
+import { countPromptTokens } from './common/tiktoken';
+
+/**
+ * text split into chunks
+ * maxLen - one chunk len. max: 3500
+ * overlapLen - The size of the before and after Text
+ * maxLen > overlapLen
+ */
+export const splitText2Chunks = ({ text, maxLen }: { text: string; maxLen: number }) => {
+  const overlapLen = Math.floor(maxLen * 0.25); // Overlap length
+
+  try {
+    const splitTexts = text.split(/(?<=[。！？；.!?;])/g);
+    const chunks: string[] = [];
+
+    let preChunk = '';
+    let chunk = '';
+    for (let i = 0; i < splitTexts.length; i++) {
+      const text = splitTexts[i];
+      chunk += text;
+      if (chunk.length > maxLen - overlapLen) {
+        preChunk += text;
+      }
+      if (chunk.length >= maxLen) {
+        chunks.push(chunk);
+        chunk = preChunk;
+        preChunk = '';
+      }
+    }
+
+    if (chunk) {
+      chunks.push(chunk);
+    }
+
+    const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0);
+
+    return {
+      chunks,
+      tokens
+    };
+  } catch (err) {
+    throw new Error(getErrText(err));
+  }
+};
+
+/* simple text, remove chinese space and extra \n */
+export const simpleText = (text: string) => {
+  text = text.replace(/([\u4e00-\u9fa5])\s+([\u4e00-\u9fa5])/g, '$1$2');
+  text = text.replace(/\n{2,}/g, '\n');
+  text = text.replace(/\s{2,}/g, ' ');
+
+  text = text.replace(/[\x00-\x08]/g, ' ');
+
+  return text;
+};

projects/app/src/utils/plugin/eventbus.ts

@@ -0,0 +1,18 @@
+export enum EventNameEnum {
+  guideClick = 'guideClick'
+}
+type EventNameType = `${EventNameEnum}`;
+
+export const event = {
+  list: new Map<EventNameType, Function>(),
+  on: function (name: EventNameType, fn: Function) {
+    this.list.set(name, fn);
+  },
+  emit: function (name: EventNameType, data: Record<string, any> = {}) {
+    const fn = this.list.get(name);
+    fn && fn(data);
+  },
+  off: function (name: EventNameType) {
+    this.list.delete(name);
+  }
+};

projects/app/src/utils/service/core/chat/index.ts

@@ -0,0 +1,15 @@
+import { ChatHistoryItemResType } from '@/types/chat';
+
+export function selectShareResponse({ responseData }: { responseData: ChatHistoryItemResType[] }) {
+  const filedList = ['moduleType', 'moduleName', 'runningTime', 'quoteList', 'question'];
+  return responseData.map((item) => {
+    const obj: Record<string, any> = {};
+    for (let key in item) {
+      if (filedList.includes(key)) {
+        // @ts-ignore
+        obj[key] = item[key];
+      }
+    }
+    return obj;
+  });
+}

projects/app/src/utils/sse.ts

@@ -0,0 +1,55 @@
+const decoder = new TextDecoder();
+
+export const parseStreamChunk = (value: BufferSource) => {
+  const chunk = decoder.decode(value);
+  const chunkLines = chunk.split('\n\n').filter((item) => item);
+  const chunkResponse = chunkLines.map((item) => {
+    const splitEvent = item.split('\n');
+    if (splitEvent.length === 2) {
+      return {
+        event: splitEvent[0].replace('event: ', ''),
+        data: splitEvent[1].replace('data: ', '')
+      };
+    }
+    return {
+      event: '',
+      data: splitEvent[0].replace('data: ', '')
+    };
+  });
+
+  return chunkResponse;
+};
+
+export class SSEParseData {
+  storeReadData = '';
+  storeEventName = '';
+
+  parse(item: { event: string; data: string }) {
+    if (item.data === '[DONE]') return { eventName: item.event, data: item.data };
+
+    if (item.event) {
+      this.storeEventName = item.event;
+    }
+
+    try {
+      const formatData = this.storeReadData + item.data;
+      const parseData = JSON.parse(formatData);
+      const eventName = this.storeEventName;
+
+      this.storeReadData = '';
+      this.storeEventName = '';
+
+      return {
+        eventName,
+        data: parseData
+      };
+    } catch (error) {
+      if (typeof item.data === 'string' && !item.data.startsWith(': ping')) {
+        this.storeReadData += item.data;
+      } else {
+        this.storeReadData = '';
+      }
+    }
+    return {};
+  }
+}

projects/app/src/utils/tools.ts

@@ -0,0 +1,113 @@
+import crypto from 'crypto';
+import dayjs from 'dayjs';
+
+/**
+ * 密码加密
+ */
+export const createHashPassword = (text: string) => {
+  const hash = crypto.createHash('sha256').update(text).digest('hex');
+  return hash;
+};
+
+/**
+ * 对象转成 query 字符串
+ */
+export const Obj2Query = (obj: Record<string, string | number>) => {
+  const queryParams = new URLSearchParams();
+  for (const key in obj) {
+    queryParams.append(key, `${obj[key]}`);
+  }
+  return queryParams.toString();
+};
+
+/**
+ * parse string to query object
+ */
+export const parseQueryString = (str: string) => {
+  const queryObject: Record<string, any> = {};
+
+  const splitStr = str.split('?');
+
+  str = splitStr[1] || splitStr[0];
+
+  // 将字符串按照 '&' 分割成键值对数组
+  const keyValuePairs = str.split('&');
+
+  // 遍历键值对数组，将每个键值对解析为对象的属性和值
+  keyValuePairs.forEach(function (keyValuePair) {
+    const pair = keyValuePair.split('=');
+    const key = decodeURIComponent(pair[0]);
+    const value = decodeURIComponent(pair[1] || '');
+
+    // 如果对象中已经存在该属性，则将值转换为数组
+    if (queryObject.hasOwnProperty(key)) {
+      if (!Array.isArray(queryObject[key])) {
+        queryObject[key] = [queryObject[key]];
+      }
+      queryObject[key].push(value);
+    } else {
+      queryObject[key] = value;
+    }
+  });
+
+  return queryObject;
+};
+
+/**
+ * 格式化时间成聊天格式
+ */
+export const formatTimeToChatTime = (time: Date) => {
+  const now = dayjs();
+  const target = dayjs(time);
+
+  // 如果传入时间小于60秒，返回刚刚
+  if (now.diff(target, 'second') < 60) {
+    return '刚刚';
+  }
+
+  // 如果时间是今天，展示几时:几秒
+  if (now.isSame(target, 'day')) {
+    return target.format('HH:mm');
+  }
+
+  // 如果是昨天，展示昨天
+  if (now.subtract(1, 'day').isSame(target, 'day')) {
+    return '昨天';
+  }
+
+  // 如果是前天，展示前天
+  if (now.subtract(2, 'day').isSame(target, 'day')) {
+    return '前天';
+  }
+
+  // 如果是今年，展示某月某日
+  if (now.isSame(target, 'year')) {
+    return target.format('M月D日');
+  }
+
+  // 如果是更久之前，展示某年某月某日
+  return target.format('YYYY/M/D');
+};
+
+export const formatFileSize = (bytes: number): string => {
+  if (bytes === 0) return '0 B';
+
+  const k = 1024;
+  const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+  const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+  return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+};
+
+export const getErrText = (err: any, def = '') => {
+  const msg: string = typeof err === 'string' ? err : err?.message || def || '';
+  msg && console.log('error =>', msg);
+  return msg;
+};
+
+export const delay = (ms: number) =>
+  new Promise((resolve) => {
+    setTimeout(() => {
+      resolve('');
+    }, ms);
+  });

projects/app/src/utils/user.ts

@@ -0,0 +1,98 @@
+import { loginOut } from '@/api/user';
+import timezones from 'timezones-list';
+import dayjs from 'dayjs';
+import utc from 'dayjs/plugin/utc';
+import timezone from 'dayjs/plugin/timezone';
+
+dayjs.extend(utc);
+dayjs.extend(timezone);
+
+const tokenKey = 'token';
+export const clearToken = () => {
+  try {
+    loginOut();
+    localStorage.removeItem(tokenKey);
+  } catch (error) {
+    error;
+  }
+};
+
+export const setToken = (token: string) => {
+  localStorage.setItem(tokenKey, token);
+};
+export const getToken = () => {
+  return localStorage.getItem(tokenKey) || '';
+};
+
+/**
+ * Returns the offset from UTC in hours for the current locale.
+ * @param {string} timeZone Timezone to get offset for
+ * @returns {number} The offset from UTC in hours.
+ *
+ * Generated by Trelent
+ */
+export const getTimezoneOffset = (timeZone: string): number => {
+  const now = new Date();
+  const tzString = now.toLocaleString('en-US', {
+    timeZone
+  });
+  const localString = now.toLocaleString('en-US');
+  const diff = (Date.parse(localString) - Date.parse(tzString)) / 3600000;
+  const offset = diff + now.getTimezoneOffset() / 60;
+  return -offset;
+};
+
+/**
+ * Returns a list of timezones sorted by their offset from UTC.
+ * @returns {object[]} A list of the given timezones sorted by their offset from UTC.
+ *
+ * Generated by Trelent
+ */
+export const timezoneList = () => {
+  const result = timezones
+    .map((timezone) => {
+      try {
+        let display = dayjs().tz(timezone.tzCode).format('Z');
+
+        return {
+          name: `(UTC${display}) ${timezone.tzCode}`,
+          value: timezone.tzCode,
+          time: getTimezoneOffset(timezone.tzCode)
+        };
+      } catch (e) {}
+    })
+    .filter((item) => item);
+
+  result.sort((a, b) => {
+    if (!a || !b) return 0;
+    if (a.time > b.time) {
+      return 1;
+    }
+
+    if (b.time > a.time) {
+      return -1;
+    }
+
+    return 0;
+  });
+
+  return [
+    {
+      name: 'UTC',
+      time: 0,
+      value: 'UTC'
+    },
+    ...result
+  ] as {
+    name: string;
+    value: string;
+    time: number;
+  }[];
+};
+
+export const getSystemTime = (timeZone: string) => {
+  const timezoneDiff = getTimezoneOffset(timeZone);
+  const now = Date.now();
+  const targetTime = now + timezoneDiff * 60 * 60 * 1000;
+  return dayjs(targetTime).format('YYYY-MM-DD HH:mm:ss');
+};

projects/app/src/utils/web/core/dataset.ts

@@ -0,0 +1,53 @@
+import { postCreateTrainingBill } from '@/api/common/bill';
+import { postChunks2Dataset } from '@/api/core/dataset/data';
+import { TrainingModeEnum } from '@/constants/plugin';
+import type { DatasetDataItemType } from '@/types/core/dataset/data';
+import { delay } from '@/utils/tools';
+
+export async function chunksUpload({
+  kbId,
+  mode,
+  chunks,
+  prompt,
+  rate = 50,
+  onUploading
+}: {
+  kbId: string;
+  mode: `${TrainingModeEnum}`;
+  chunks: DatasetDataItemType[];
+  prompt?: string;
+  rate?: number;
+  onUploading?: (insertLen: number, total: number) => void;
+}) {
+  // create training bill
+  const billId = await postCreateTrainingBill({ name: 'dataset.Training Name' });
+
+  async function upload(data: DatasetDataItemType[]) {
+    return postChunks2Dataset({
+      kbId,
+      data,
+      mode,
+      prompt,
+      billId
+    });
+  }
+
+  let successInsert = 0;
+  let retryTimes = 10;
+  for (let i = 0; i < chunks.length; i += rate) {
+    try {
+      const { insertLen } = await upload(chunks.slice(i, i + rate));
+      onUploading && onUploading(i + rate, chunks.length);
+      successInsert += insertLen;
+    } catch (error) {
+      if (retryTimes === 0) {
+        return Promise.reject(error);
+      }
+      await delay(1000);
+      retryTimes--;
+      i -= rate;
+    }
+  }
+
+  return { insertLen: successInsert };
+}

projects/app/src/utils/web/file.ts

@@ -0,0 +1,261 @@
+import mammoth from 'mammoth';
+import Papa from 'papaparse';
+import { uploadImg, postUploadFiles, getFileViewUrl } from '@/api/support/file';
+/**
+ * upload file to mongo gridfs
+ */
+export const uploadFiles = (
+  files: File[],
+  metadata: Record<string, any> = {},
+  percentListen?: (percent: number) => void
+) => {
+  const form = new FormData();
+  form.append('metadata', JSON.stringify(metadata));
+  files.forEach((file) => {
+    form.append('file', file, encodeURIComponent(file.name));
+  });
+  return postUploadFiles(form, (e) => {
+    if (!e.total) return;
+
+    const percent = Math.round((e.loaded / e.total) * 100);
+    percentListen && percentListen(percent);
+  });
+};
+
+/**
+ * 读取 txt 文件内容
+ */
+export const readTxtContent = (file: File) => {
+  return new Promise((resolve: (_: string) => void, reject) => {
+    try {
+      const reader = new FileReader();
+      reader.onload = () => {
+        resolve(reader.result as string);
+      };
+      reader.onerror = (err) => {
+        console.log('error txt read:', err);
+        reject('读取 txt 文件失败');
+      };
+      reader.readAsText(file);
+    } catch (error) {
+      reject('浏览器不支持文件内容读取');
+    }
+  });
+};
+
+/**
+ * 读取 pdf 内容
+ */
+export const readPdfContent = (file: File) =>
+  new Promise<string>((resolve, reject) => {
+    try {
+      const pdfjsLib = window['pdfjs-dist/build/pdf'];
+      pdfjsLib.workerSrc = '/js/pdf.worker.js';
+
+      const readPDFPage = async (doc: any, pageNo: number) => {
+        const page = await doc.getPage(pageNo);
+        const tokenizedText = await page.getTextContent();
+
+        const pageText = tokenizedText.items
+          .map((token: any) => token.str)
+          .filter((item: string) => item)
+          .join('');
+        return pageText;
+      };
+
+      let reader = new FileReader();
+      reader.readAsArrayBuffer(file);
+      reader.onload = async (event) => {
+        if (!event?.target?.result) return reject('解析 PDF 失败');
+        try {
+          const doc = await pdfjsLib.getDocument(event.target.result).promise;
+          const pageTextPromises = [];
+          for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
+            pageTextPromises.push(readPDFPage(doc, pageNo));
+          }
+          const pageTexts = await Promise.all(pageTextPromises);
+          resolve(pageTexts.join('\n'));
+        } catch (err) {
+          console.log(err, 'pdf load error');
+          reject('解析 PDF 失败');
+        }
+      };
+      reader.onerror = (err) => {
+        console.log(err, 'pdf load error');
+        reject('解析 PDF 失败');
+      };
+    } catch (error) {
+      reject('浏览器不支持文件内容读取');
+    }
+  });
+
+/**
+ * 读取doc
+ */
+export const readDocContent = (file: File) =>
+  new Promise<string>((resolve, reject) => {
+    try {
+      const reader = new FileReader();
+      reader.readAsArrayBuffer(file);
+      reader.onload = async ({ target }) => {
+        if (!target?.result) return reject('读取 doc 文件失败');
+        try {
+          const res = await mammoth.extractRawText({
+            arrayBuffer: target.result as ArrayBuffer
+          });
+          resolve(res?.value);
+        } catch (error) {
+          window.umami?.track('wordReadError', {
+            err: error?.toString()
+          });
+          console.log('error doc read:', error);
+
+          reject('读取 doc 文件失败, 请转换成 PDF');
+        }
+      };
+      reader.onerror = (err) => {
+        window.umami?.track('wordReadError', {
+          err: err?.toString()
+        });
+        console.log('error doc read:', err);
+
+        reject('读取 doc 文件失败');
+      };
+    } catch (error) {
+      reject('浏览器不支持文件内容读取');
+    }
+  });
+
+/**
+ * 读取csv
+ */
+export const readCsvContent = async (file: File) => {
+  try {
+    const textArr = await readTxtContent(file);
+    const csvArr = Papa.parse(textArr).data as string[][];
+    if (csvArr.length === 0) {
+      throw new Error('csv 解析失败');
+    }
+    return {
+      header: csvArr.shift() as string[],
+      data: csvArr.map((item) => item)
+    };
+  } catch (error) {
+    return Promise.reject('解析 csv 文件失败');
+  }
+};
+
+/**
+ * file download
+ */
+export const fileDownload = ({
+  text,
+  type,
+  filename
+}: {
+  text: string;
+  type: string;
+  filename: string;
+}) => {
+  // 导出为文件
+  const blob = new Blob([`\uFEFF${text}`], { type: `${type};charset=utf-8;` });
+
+  // 创建下载链接
+  const downloadLink = document.createElement('a');
+  downloadLink.href = window.URL.createObjectURL(blob);
+  downloadLink.download = filename;
+
+  // 添加链接到页面并触发下载
+  document.body.appendChild(downloadLink);
+  downloadLink.click();
+  document.body.removeChild(downloadLink);
+};
+
+export async function getFileAndOpen(fileId: string) {
+  const url = await getFileViewUrl(fileId);
+  const asPath = `${location.origin}${url}`;
+  window.open(asPath, '_blank');
+}
+
+export const fileToBase64 = (file: File) => {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(file);
+    reader.onload = () => resolve(reader.result);
+    reader.onerror = (error) => reject(error);
+  });
+};
+
+/**
+ * compress image. response base64
+ * @param maxSize The max size of the compressed image
+ */
+export const compressImg = ({
+  file,
+  maxW = 200,
+  maxH = 200,
+  maxSize = 1024 * 100
+}: {
+  file: File;
+  maxW?: number;
+  maxH?: number;
+  maxSize?: number;
+}) =>
+  new Promise<string>((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(file);
+    reader.onload = async () => {
+      const img = new Image();
+      // @ts-ignore
+      img.src = reader.result;
+      img.onload = async () => {
+        let width = img.width;
+        let height = img.height;
+
+        if (width > height) {
+          if (width > maxW) {
+            height *= maxW / width;
+            width = maxW;
+          }
+        } else {
+          if (height > maxH) {
+            width *= maxH / height;
+            height = maxH;
+          }
+        }
+
+        const canvas = document.createElement('canvas');
+        canvas.width = width;
+        canvas.height = height;
+        const ctx = canvas.getContext('2d');
+
+        if (!ctx) {
+          return reject('压缩图片异常');
+        }
+
+        ctx.drawImage(img, 0, 0, width, height);
+        const compressedDataUrl = canvas.toDataURL(file.type, 0.8);
+        // 移除 canvas 元素
+        canvas.remove();
+
+        if (compressedDataUrl.length > maxSize) {
+          return reject('图片太大了');
+        }
+
+        const src = await (async () => {
+          try {
+            const src = await uploadImg(compressedDataUrl);
+            return src;
+          } catch (error) {
+            return compressedDataUrl;
+          }
+        })();
+
+        resolve(src);
+      };
+    };
+    reader.onerror = (err) => {
+      console.log(err);
+      reject('压缩图片异常');
+    };
+  });

projects/app/src/utils/web/i18n.ts

@@ -0,0 +1,37 @@
+import { serverSideTranslations } from 'next-i18next/serverSideTranslations';
+import Cookies from 'js-cookie';
+
+export const LANG_KEY = 'NEXT_LOCALE_LANG';
+export enum LangEnum {
+  'zh' = 'zh',
+  'en' = 'en'
+}
+export const langMap = {
+  [LangEnum.en]: {
+    label: 'English',
+    icon: 'language_en'
+  },
+  [LangEnum.zh]: {
+    label: '简体中文',
+    icon: 'language_zh'
+  }
+};
+
+export const setLangStore = (value: `${LangEnum}`) => {
+  return Cookies.set(LANG_KEY, value, { expires: 7, sameSite: 'None', secure: true });
+};
+
+export const getLangStore = () => {
+  return (Cookies.get(LANG_KEY) as `${LangEnum}`) || LangEnum.zh;
+};
+
+export const serviceSideProps = (content: any) => {
+  const acceptLanguage = (content.req.headers['accept-language'] as string) || '';
+  const acceptLanguageList = acceptLanguage.split(/,|;/g);
+  // @ts-ignore
+  const firstLang = acceptLanguageList.find((lang) => langMap[lang]);
+
+  const language = content.req.cookies[LANG_KEY] || firstLang || 'zh';
+
+  return serverSideTranslations(language, undefined, null, content.locales);
+};

projects/app/src/utils/web/voice.ts

@@ -0,0 +1,28 @@
+export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window;
+/**
+ * voice broadcast
+ */
+export const voiceBroadcast = ({ text }: { text: string }) => {
+  window.speechSynthesis?.cancel();
+  const msg = new SpeechSynthesisUtterance(text);
+  const voices = window.speechSynthesis?.getVoices?.(); // 获取语言包
+  const voice = voices.find((item) => {
+    return item.name === 'Microsoft Yaoyao - Chinese (Simplified, PRC)';
+  });
+  if (voice) {
+    msg.voice = voice;
+  }
+
+  window.speechSynthesis?.speak(msg);
+
+  msg.onerror = (e) => {
+    console.log(e);
+  };
+
+  return {
+    cancel: () => window.speechSynthesis?.cancel()
+  };
+};
+export const cancelBroadcast = () => {
+  window.speechSynthesis?.cancel();
+};