fix: img compress (#546)

* Fix: Update Custom Chat Title Based on chatId

* remove `new:true` option

* Update update.ts

---------

Co-authored-by: Archer <545436317@qq.com>

Dockerfile

@@ -1,7 +1,7 @@
 # Install dependencies only when needed
 FROM node:18.15-alpine AS deps
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk add --no-cache libc6-compat && npm install -g pnpm
+RUN apk add libc6-compat && npm install -g pnpm
 WORKDIR /app
 
 ARG name

docSite/content/docs/custom-models/chatglm2-m3e.md

@@ -59,7 +59,7 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
 
 ## 接入 FastGPT
 
-修改 config.json 配置文件，在 VectorModels 中加入 chatglm2 和 M3E 模型：
+修改 config.json 配置文件，在 ChatModels 中加入 chatglm2, 在 VectorModels 中加入 M3E 模型：
 
 ```json
 "ChatModels": [

docSite/content/docs/custom-models/chatglm2.md

@@ -99,7 +99,7 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
 
 ## 接入 FastGPT
 
-修改 config.json 配置文件，在 VectorModels 中加入 chatglm2 模型：
+修改 config.json 配置文件，在 ChatModels 中加入 chatglm2 模型：
 
 ```json
 "ChatModels": [
@@ -107,10 +107,11 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
   {
     "model": "chatglm2",
     "name": "chatglm2",
-    "maxToken": 8000,
-    "price": 0,
-    "quoteMaxToken": 4000,
-    "maxTemperature": 1.2,
+    "maxContext": 4000,
+    "maxResponse": 4000,
+    "quoteMaxToken": 2000,
+    "maxTemperature": 1,
+    "vision": false, 
     "defaultSystemChatPrompt": ""
   }
 ]

docSite/content/docs/development/openapi/dataset.md

@@ -21,7 +21,9 @@ weight: 563
 curl --location --request POST 'https://fastgpt.run/api/support/wallet/bill/createTrainingBill' \
 --header 'Authorization: Bearer {{apikey}}' \
 --header 'Content-Type: application/json' \
---data-raw ''
+--data-raw '{
+    "name": "可选，自定义订单名称，例如：文档训练-fastgpt.docx"
+}'
 ```
 
 **响应结果**

docSite/content/docs/installation/docker.md

@@ -86,7 +86,7 @@ curl -O https://raw.githubusercontent.com/labring/FastGPT/main/projects/app/data
 
 ## 三、启动容器
 
-修改`docker-compose.yml`中的`OPENAI_BASE_URL`和`CHAT_API_KEY`即可，对应为 API 的地址和 key。
+修改`docker-compose.yml`中的`OPENAI_BASE_URL`和`CHAT_API_KEY`即可，对应为 API 的地址(别忘记加/v1)和 key。
 
 ```bash
 # 在 docker-compose.yml 同级目录下执行

docSite/content/docs/installation/upgrading/463.md

@@ -0,0 +1,33 @@
+---
+title: 'V4.6.3(需要初始化)'
+description: 'FastGPT V4.6.3'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 833
+---
+
+## 1。执行初始化 API
+
+发起 1 个 HTTP 请求 ({{rootkey}} 替换成环境变量里的 `rootkey`，{{host}} 替换成自己域名)
+
+1. https://xxxxx/api/admin/initv463
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv463' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化说明：
+1. 初始化Mongo 中 dataset，collection 和 data 的部分字段
+
+## V4.6.3 功能介绍
+
+1. 商业版新增 - web站点同步
+2. 新增 - 集合元数据记录
+3. 优化 - url 读取内容
+4. 优化 - 流读取文件，防止内存溢出
+5. 优化 - 4v模型自动将 url 转 base64，本地也可调试
+6. 优化 - 图片压缩等级
+7. 修复 - 图片压缩失败报错，防止文件读取过程卡死。

packages/global/common/file/api.d.ts

@@ -0,0 +1,8 @@
+export type UrlFetchParams = {
+  urlList: string[];
+  selector?: string;
+};
+export type UrlFetchResponse = {
+  url: string;
+  content: string;
+}[];

packages/global/common/file/tools.ts

@@ -1,3 +1,8 @@
+import axios from 'axios';
+import { UrlFetchParams, UrlFetchResponse } from './api.d';
+import { htmlToMarkdown } from '../string/markdown';
+import * as cheerio from 'cheerio';
+
 export const formatFileSize = (bytes: number): string => {
   if (bytes === 0) return '0 B';
 
@@ -7,3 +12,84 @@ export const formatFileSize = (bytes: number): string => {
 
   return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
 };
+
+export const cheerioToHtml = ({
+  fetchUrl,
+  $,
+  selector
+}: {
+  fetchUrl: string;
+  $: cheerio.CheerioAPI;
+  selector?: string;
+}) => {
+  // get origin url
+  const originUrl = new URL(fetchUrl).origin;
+
+  // remove i element
+  $('i,script').remove();
+
+  // remove empty a element
+  $('a')
+    .filter((i, el) => {
+      return $(el).text().trim() === '' && $(el).children().length === 0;
+    })
+    .remove();
+
+  // if link,img startWith /, add origin url
+  $('a').each((i, el) => {
+    const href = $(el).attr('href');
+    if (href && href.startsWith('/')) {
+      $(el).attr('href', originUrl + href);
+    }
+  });
+  $('img').each((i, el) => {
+    const src = $(el).attr('src');
+    if (src && src.startsWith('/')) {
+      $(el).attr('src', originUrl + src);
+    }
+  });
+
+  return $(selector || 'body').html();
+};
+export const urlsFetch = async ({
+  urlList,
+  selector
+}: UrlFetchParams): Promise<UrlFetchResponse> => {
+  urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
+
+  const response = (
+    await Promise.all(
+      urlList.map(async (url) => {
+        try {
+          const fetchRes = await axios.get(url, {
+            timeout: 30000
+          });
+
+          const $ = cheerio.load(fetchRes.data);
+
+          const md = htmlToMarkdown(
+            cheerioToHtml({
+              fetchUrl: url,
+              $,
+              selector
+            })
+          );
+
+          return {
+            url,
+            content: md
+          };
+        } catch (error) {
+          console.log(error, 'fetch error');
+
+          return {
+            url,
+            content: ''
+          };
+        }
+      })
+    )
+  ).filter((item) => item.content);
+
+  return response;
+};

packages/global/common/plugin/types/pluginRes.d.ts

@@ -1,4 +0,0 @@
-export type FetchResultItem = {
-  url: string;
-  content: string;
-};

packages/global/common/string/markdown.ts

@@ -0,0 +1,97 @@
+import { simpleText } from './tools';
+import { NodeHtmlMarkdown } from 'node-html-markdown';
+
+/* Delete redundant text in markdown */
+export const simpleMarkdownText = (rawText: string) => {
+  rawText = simpleText(rawText);
+
+  // Remove a line feed from a hyperlink or picture
+  rawText = rawText.replace(/\[([^\]]+)\]\((.+?)\)/g, (match, linkText, url) => {
+    const cleanedLinkText = linkText.replace(/\n/g, ' ').trim();
+
+    if (!url) {
+      return '';
+    }
+
+    return `[${cleanedLinkText}](${url})`;
+  });
+
+  // replace special \.* ……
+  const reg1 = /\\([-.!`_(){}\[\]])/g;
+  if (reg1.test(rawText)) {
+    rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1');
+  }
+
+  // replace \\n
+  rawText = rawText.replace(/\\\\n/g, '\\n');
+
+  // Remove headings and code blocks front spaces
+  ['####', '###', '##', '#', '```', '~~~'].forEach((item) => {
+    const reg = new RegExp(`\\n\\s*${item}`, 'g');
+    if (reg.test(rawText)) {
+      rawText = rawText.replace(new RegExp(`\\n\\s*(${item})`, 'g'), '\n$1');
+    }
+  });
+
+  return rawText.trim();
+};
+
+/* html string to markdown */
+export const htmlToMarkdown = (html?: string | null) => {
+  if (!html) return '';
+
+  const surround = (source: string, surroundStr: string) => `${surroundStr}${source}${surroundStr}`;
+
+  const nhm = new NodeHtmlMarkdown(
+    {
+      codeFence: '```',
+      codeBlockStyle: 'fenced',
+      ignore: ['i', 'script']
+    },
+    {
+      code: ({ node, parent, options: { codeFence, codeBlockStyle }, visitor }) => {
+        const isCodeBlock = ['PRE', 'WRAPPED-PRE'].includes(parent?.tagName!);
+
+        if (!isCodeBlock) {
+          return {
+            spaceIfRepeatingChar: true,
+            noEscape: true,
+            postprocess: ({ content }) => {
+              // Find longest occurring sequence of running backticks and add one more (so content is escaped)
+              const delimiter =
+                '`' + (content.match(/`+/g)?.sort((a, b) => b.length - a.length)?.[0] || '');
+              const padding = delimiter.length > 1 ? ' ' : '';
+
+              return surround(surround(content, padding), delimiter);
+            }
+          };
+        }
+
+        /* Handle code block */
+        if (codeBlockStyle === 'fenced') {
+          const language =
+            node.getAttribute('class')?.match(/language-(\S+)/)?.[1] ||
+            parent?.getAttribute('class')?.match(/language-(\S+)/)?.[1] ||
+            '';
+
+          return {
+            noEscape: true,
+            prefix: `${codeFence}${language}\n`,
+            postfix: `\n${codeFence}\n`,
+            childTranslators: visitor.instance.codeBlockTranslators
+          };
+        }
+
+        return {
+          noEscape: true,
+          postprocess: ({ content }) => content.replace(/^/gm, '    '),
+          childTranslators: visitor.instance.codeBlockTranslators
+        };
+      }
+    }
+  );
+
+  const markdown = nhm.translate(html).trim();
+
+  return simpleMarkdownText(markdown);
+};

packages/global/common/string/textSplitter.ts

@@ -15,11 +15,18 @@ export const splitText2Chunks = (props: {
 }): {
   chunks: string[];
   tokens: number;
+  overlapRatio?: number;
 } => {
-  const { text = '', chunkLen, overlapRatio = 0.2 } = props;
+  let { text = '', chunkLen, overlapRatio = 0.2 } = props;
   const splitMarker = 'SPLIT_HERE_SPLIT_HERE';
+  const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER';
   const overlapLen = Math.round(chunkLen * overlapRatio);
 
+  // replace code block all \n to codeBlockMarker
+  text = text.replace(/(```[\s\S]*?```|~~~[\s\S]*?~~~)/g, function (match) {
+    return match.replace(/\n/g, codeBlockMarker);
+  });
+
   // The larger maxLen is, the next sentence is less likely to trigger splitting
   const stepReges: { reg: RegExp; maxLen: number }[] = [
     { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
@@ -27,8 +34,8 @@ export const splitText2Chunks = (props: {
     { reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
     { reg: /^(####\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
 
-    { reg: /([\n]{2})/g, maxLen: chunkLen * 1.4 },
-    { reg: /([\n](?![\*\-|>`0-9]))/g, maxLen: chunkLen * 1.8 }, // (?![\*\-|>`0-9]): markdown special char
+    { reg: /([\n](`))/g, maxLen: chunkLen * 4 }, // code block
+    { reg: /([\n](?![\*\-|>0-9]))/g, maxLen: chunkLen * 1.8 }, // (?![\*\-|>`0-9]): markdown special char
     { reg: /([\n])/g, maxLen: chunkLen * 1.4 },
 
     { reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.4 },
@@ -38,9 +45,15 @@ export const splitText2Chunks = (props: {
     { reg: /([，]|,\s)/g, maxLen: chunkLen * 2 }
   ];
 
+  // if use markdown title split, Separate record title title
   const getSplitTexts = ({ text, step }: { text: string; step: number }) => {
     if (step >= stepReges.length) {
-      return [text];
+      return [
+        {
+          text,
+          title: ''
+        }
+      ];
     }
     const isMarkdownSplit = step <= 3;
     const { reg } = stepReges[step];
@@ -49,7 +62,17 @@ export const splitText2Chunks = (props: {
       .replace(reg, isMarkdownSplit ? `${splitMarker}$1` : `$1${splitMarker}`)
       .split(`${splitMarker}`)
       .filter((part) => part.trim());
-    return splitTexts;
+
+    return splitTexts
+      .map((text) => {
+        const matchTitle = isMarkdownSplit ? text.match(reg)?.[0] || '' : '';
+
+        return {
+          text: isMarkdownSplit ? text.replace(matchTitle, '') : text,
+          title: matchTitle
+        };
+      })
+      .filter((item) => item.text.trim());
   };
 
   const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
@@ -63,7 +86,7 @@ export const splitText2Chunks = (props: {
     let overlayText = '';
 
     for (let i = splitTexts.length - 1; i >= 0; i--) {
-      const currentText = splitTexts[i];
+      const currentText = splitTexts[i].text;
       const newText = currentText + overlayText;
       const newTextLen = newText.length;
 
@@ -83,12 +106,16 @@ export const splitText2Chunks = (props: {
   const splitTextRecursively = ({
     text = '',
     step,
-    lastText
+    lastText,
+    mdTitle = ''
   }: {
     text: string;
     step: number;
     lastText: string;
+    mdTitle: string;
   }): string[] => {
+    const isMarkdownSplit = step <= 3;
+
     // mini text
     if (text.length <= chunkLen) {
       return [text];
@@ -102,7 +129,7 @@ export const splitText2Chunks = (props: {
       // use slice-chunkLen to split text
       const chunks: string[] = [];
       for (let i = 0; i < text.length; i += chunkLen - overlapLen) {
-        chunks.push(text.slice(i, i + chunkLen));
+        chunks.push(`${mdTitle}${text.slice(i, i + chunkLen)}`);
       }
       return chunks;
     }
@@ -115,7 +142,10 @@ export const splitText2Chunks = (props: {
 
     const chunks: string[] = [];
     for (let i = 0; i < splitTexts.length; i++) {
-      const currentText = splitTexts[i];
+      const item = splitTexts[i];
+      const currentTitle = `${mdTitle}${item.title}`;
+
+      const currentText = item.text;
       const currentTextLen = currentText.length;
       const lastTextLen = lastText.length;
       const newText = lastText + currentText;
@@ -125,9 +155,10 @@ export const splitText2Chunks = (props: {
       if (newTextLen > maxLen) {
         // lastText greater minChunkLen, direct push it to chunks, not add to next chunk. (large lastText)
         if (lastTextLen > minChunkLen) {
-          chunks.push(lastText);
+          chunks.push(`${currentTitle}${lastText}`);
           lastText = getOneTextOverlapText({ text: lastText, step }); // next chunk will start with overlayText
           i--;
+
           continue;
         }
 
@@ -135,11 +166,12 @@ export const splitText2Chunks = (props: {
         const innerChunks = splitTextRecursively({
           text: newText,
           step: step + 1,
-          lastText: ''
+          lastText: '',
+          mdTitle: currentTitle
         });
         const lastChunk = innerChunks[innerChunks.length - 1];
         // last chunk is too small, concat it to lastText
-        if (lastChunk.length < minChunkLen) {
+        if (!isMarkdownSplit && lastChunk.length < minChunkLen) {
           chunks.push(...innerChunks.slice(0, -1));
           lastText = lastChunk;
         } else {
@@ -156,10 +188,11 @@ export const splitText2Chunks = (props: {
       // size less than chunkLen, push text to last chunk. now, text definitely less than maxLen
       lastText = newText;
 
-      // If the chunk size reaches, add a chunk
-      if (newTextLen >= chunkLen) {
-        chunks.push(lastText);
-        lastText = getOneTextOverlapText({ text: lastText, step });
+      // markdown paragraph block: Direct addition; If the chunk size reaches, add a chunk
+      if (isMarkdownSplit || newTextLen >= chunkLen) {
+        chunks.push(`${currentTitle}${lastText}`);
+
+        lastText = isMarkdownSplit ? '' : getOneTextOverlapText({ text: lastText, step });
       }
     }
 
@@ -168,7 +201,7 @@ export const splitText2Chunks = (props: {
       if (lastText.length < chunkLen * 0.4) {
         chunks[chunks.length - 1] = chunks[chunks.length - 1] + lastText;
       } else {
-        chunks.push(lastText);
+        chunks.push(`${mdTitle}${lastText}`);
       }
     }
 
@@ -179,8 +212,9 @@ export const splitText2Chunks = (props: {
     const chunks = splitTextRecursively({
       text,
       step: 0,
-      lastText: ''
-    });
+      lastText: '',
+      mdTitle: ''
+    }).map((chunk) => chunk.replaceAll(codeBlockMarker, '\n')); // restore code block
 
     const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0);
 

packages/global/common/string/time.ts

@@ -0,0 +1,3 @@
+import dayjs from 'dayjs';
+
+export const formatTime2YMDHM = (time: Date) => dayjs(time).format('YYYY-MM-DD HH:mm');

packages/global/common/system/types/index.d.ts

@@ -2,16 +2,15 @@ export type FeConfigsType = {
   show_emptyChat?: boolean;
   show_register?: boolean;
   show_appStore?: boolean;
-  show_contact?: boolean;
   show_git?: boolean;
   show_pay?: boolean;
   show_openai_account?: boolean;
   show_promotion?: boolean;
   hide_app_flow?: boolean;
+  concatMd?: string;
   docUrl?: string;
   openAPIDocUrl?: string;
   systemTitle?: string;
-  authorText?: string;
   googleClientVerKey?: string;
   isPlus?: boolean;
   oauth?: {

packages/global/common/system/utils.ts

@@ -0,0 +1,6 @@
+export const delay = (ms: number) =>
+  new Promise((resolve) => {
+    setTimeout(() => {
+      resolve('');
+    }, ms);
+  });

packages/global/core/ai/api.d.ts

@@ -2,4 +2,4 @@ export type PostReRankProps = {
   query: string;
   inputs: { id: string; text: string }[];
 };
-export type PostReRankResponse = { id: string; score: number }[];
+export type PostReRankResponse = { id: string; score?: number }[];

packages/global/core/ai/model.d.ts

@@ -26,6 +26,14 @@ export type VectorModelItemType = {
   maxToken: number;
 };
 
+export type ReRankModelItemType = {
+  model: string;
+  name: string;
+  price: number;
+  requestUrl?: string;
+  requestAuth?: string;
+};
+
 export type AudioSpeechModelType = {
   model: string;
   name: string;

packages/global/core/ai/model.ts

@@ -4,7 +4,8 @@ import type {
   FunctionModelItemType,
   VectorModelItemType,
   AudioSpeechModelType,
-  WhisperModelType
+  WhisperModelType,
+  ReRankModelItemType
 } from './model.d';
 
 export const defaultChatModels: ChatModelItemType[] = [
@@ -117,6 +118,8 @@ export const defaultVectorModels: VectorModelItemType[] = [
   }
 ];
 
+export const defaultReRankModels: ReRankModelItemType[] = [];
+
 export const defaultAudioSpeechModels: AudioSpeechModelType[] = [
   {
     model: 'tts-1',

packages/global/core/app/type.d.ts

@@ -4,6 +4,7 @@ import { PermissionTypeEnum } from '../../support/permission/constant';
 import type { AIChatModuleProps, DatasetModuleProps } from '../module/node/type.d';
 import { VariableInputEnum } from '../module/constants';
 import { SelectedDatasetType } from '../module/api';
+import { DatasetSearchModeEnum } from '../dataset/constant';
 
 export interface AppSchema {
   _id: string;
@@ -18,6 +19,7 @@ export interface AppSchema {
   updateTime: number;
   modules: ModuleItemType[];
   permission: `${PermissionTypeEnum}`;
+  inited?: boolean;
 }
 
 export type AppListItemType = {
@@ -62,7 +64,7 @@ export type AppSimpleEditFormType = {
     datasets: SelectedDatasetType;
     similarity: number;
     limit: number;
-    rerank: boolean;
+    searchMode: `${DatasetSearchModeEnum}`;
     searchEmptyText: string;
   };
   userGuide: {
@@ -106,7 +108,7 @@ export type AppSimpleEditConfigTemplateType = {
       datasets?: boolean;
       similarity?: boolean;
       limit?: boolean;
-      rerank?: boolean;
+      searchMode: `${DatasetSearchModeEnum}`;
       searchEmptyText?: boolean;
     };
     userGuide?: {

packages/global/core/app/utils.ts

@@ -5,6 +5,7 @@ import type { FlowNodeInputItemType } from '../module/node/type.d';
 import { getGuideModule, splitGuideModule } from '../module/utils';
 import { defaultChatModels } from '../ai/model';
 import { ModuleItemType } from '../module/type.d';
+import { DatasetSearchModeEnum } from '../dataset/constant';
 
 export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEditFormType => {
   const defaultChatModel = defaultChatModels[0];
@@ -25,7 +26,7 @@ export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEd
       similarity: 0.4,
       limit: 5,
       searchEmptyText: '',
-      rerank: false
+      searchMode: DatasetSearchModeEnum.embedding
     },
     userGuide: {
       welcomeText: '',
@@ -76,7 +77,7 @@ export const appModules2Form = ({
       );
       defaultAppForm.aiSettings.quotePrompt = findInputValueByKey(
         module.inputs,
-        ModuleInputKeyEnum.aiChatQuoteTemplate
+        ModuleInputKeyEnum.aiChatQuotePrompt
       );
     } else if (module.flowType === FlowNodeTypeEnum.datasetSearchNode) {
       defaultAppForm.dataset.datasets = findInputValueByKey(
@@ -91,10 +92,9 @@ export const appModules2Form = ({
         module.inputs,
         ModuleInputKeyEnum.datasetLimit
       );
-      defaultAppForm.dataset.rerank = findInputValueByKey(
-        module.inputs,
-        ModuleInputKeyEnum.datasetStartReRank
-      );
+      defaultAppForm.dataset.searchMode =
+        findInputValueByKey(module.inputs, ModuleInputKeyEnum.datasetSearchMode) ||
+        DatasetSearchModeEnum.embedding;
 
       // empty text
       const emptyOutputs =

packages/global/core/chat/constants.ts

@@ -5,7 +5,6 @@ export enum ChatRoleEnum {
   Function = 'Function',
   Tool = 'Tool'
 }
-
 export const ChatRoleMap = {
   [ChatRoleEnum.System]: {
     name: '系统提示词'
@@ -30,7 +29,6 @@ export enum ChatSourceEnum {
   share = 'share',
   api = 'api'
 }
-
 export const ChatSourceMap = {
   [ChatSourceEnum.test]: {
     name: 'chat.logs.test'

packages/global/core/chat/type.d.ts

@@ -4,6 +4,7 @@ import { ChatRoleEnum, ChatSourceEnum } from './constants';
 import { FlowNodeTypeEnum } from '../module/node/constant';
 import { ModuleOutputKeyEnum } from '../module/constants';
 import { AppSchema } from '../app/type';
+import { DatasetSearchModeEnum } from '../dataset/constant';
 
 export type ChatSchema = {
   _id: string;
@@ -94,6 +95,7 @@ export type moduleDispatchResType = {
   // dataset search
   similarity?: number;
   limit?: number;
+  searchMode?: `${DatasetSearchModeEnum}`;
 
   // cq
   cqList?: ClassifyQuestionAgentItemType[];

packages/global/core/dataset/api.d.ts

@@ -1,8 +1,32 @@
-import { DatasetDataIndexItemType } from './type';
+import { DatasetDataIndexItemType, DatasetSchemaType } from './type';
+import { DatasetCollectionTrainingModeEnum, DatasetCollectionTypeEnum } from './constant';
+import type { LLMModelItemType } from '../ai/model.d';
 
 /* ================= dataset ===================== */
+export type DatasetUpdateBody = {
+  id: string;
+  parentId?: string;
+  tags?: string[];
+  name?: string;
+  avatar?: string;
+  permission?: DatasetSchemaType['permission'];
+  agentModel?: LLMModelItemType;
+  websiteConfig?: DatasetSchemaType['websiteConfig'];
+  status?: DatasetSchemaType['status'];
+};
 
 /* ================= collection ===================== */
+export type CreateDatasetCollectionParams = {
+  datasetId: string;
+  parentId?: string;
+  name: string;
+  type: `${DatasetCollectionTypeEnum}`;
+  trainingType?: `${DatasetCollectionTrainingModeEnum}`;
+  chunkSize?: number;
+  fileId?: string;
+  rawLink?: string;
+  metadata?: Record<string, any>;
+};
 
 /* ================= data ===================== */
 export type PgSearchRawType = {
@@ -18,3 +42,8 @@ export type PushDatasetDataChunkProps = {
   a?: string; // bonus content
   indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
 };
+
+export type PostWebsiteSyncParams = {
+  datasetId: string;
+  billId: string;
+};

packages/global/core/dataset/constant.ts

@@ -1,41 +1,81 @@
 export const PgDatasetTableName = 'modeldata';
 
+/* ------------ dataset -------------- */
 export enum DatasetTypeEnum {
   folder = 'folder',
-  dataset = 'dataset'
+  dataset = 'dataset',
+  websiteDataset = 'websiteDataset' // depp link
 }
-
 export const DatasetTypeMap = {
   [DatasetTypeEnum.folder]: {
-    name: 'folder'
+    icon: 'core/dataset/folderDataset',
+    label: 'core.dataset.Folder Dataset',
+    collectionLabel: 'common.Folder'
   },
   [DatasetTypeEnum.dataset]: {
-    name: 'dataset'
+    icon: 'core/dataset/commonDataset',
+    label: 'core.dataset.Common Dataset',
+    collectionLabel: 'common.File'
+  },
+  [DatasetTypeEnum.websiteDataset]: {
+    icon: 'core/dataset/websiteDataset',
+    label: 'core.dataset.Website Dataset',
+    collectionLabel: 'common.Website'
   }
 };
 
+export enum DatasetStatusEnum {
+  active = 'active',
+  syncing = 'syncing'
+}
+export const DatasetStatusMap = {
+  [DatasetStatusEnum.active]: {
+    label: 'core.dataset.status.active'
+  },
+  [DatasetStatusEnum.syncing]: {
+    label: 'core.dataset.status.syncing'
+  }
+};
+
+/* ------------ collection -------------- */
 export enum DatasetCollectionTypeEnum {
-  file = 'file',
   folder = 'folder',
-  link = 'link',
+  file = 'file',
+  link = 'link', // one link
   virtual = 'virtual'
 }
-
 export const DatasetCollectionTypeMap = {
-  [DatasetCollectionTypeEnum.file]: {
-    name: 'dataset.file'
-  },
   [DatasetCollectionTypeEnum.folder]: {
-    name: 'dataset.folder'
+    name: 'core.dataset.folder'
+  },
+  [DatasetCollectionTypeEnum.file]: {
+    name: 'core.dataset.file'
   },
   [DatasetCollectionTypeEnum.link]: {
-    name: 'dataset.link'
+    name: 'core.dataset.link'
   },
   [DatasetCollectionTypeEnum.virtual]: {
-    name: 'dataset.Virtual File'
+    name: 'core.dataset.Virtual File'
+  }
+};
+export enum DatasetCollectionTrainingModeEnum {
+  manual = 'manual',
+  chunk = 'chunk',
+  qa = 'qa'
+}
+export const DatasetCollectionTrainingTypeMap = {
+  [DatasetCollectionTrainingModeEnum.manual]: {
+    label: 'core.dataset.collection.training.type manual'
+  },
+  [DatasetCollectionTrainingModeEnum.chunk]: {
+    label: 'core.dataset.collection.training.type chunk'
+  },
+  [DatasetCollectionTrainingModeEnum.qa]: {
+    label: 'core.dataset.collection.training.type qa'
   }
 };
 
+/* ------------ data -------------- */
 export enum DatasetDataIndexTypeEnum {
   chunk = 'chunk',
   qa = 'qa',
@@ -61,29 +101,47 @@ export const DatasetDataIndexTypeMap = {
   }
 };
 
+/* ------------ training -------------- */
 export enum TrainingModeEnum {
-  'chunk' = 'chunk',
-  'qa' = 'qa'
-  // 'hypothetical' = 'hypothetical',
-  // 'summary' = 'summary',
-  // 'multipleIndex' = 'multipleIndex'
+  chunk = 'chunk',
+  qa = 'qa'
 }
+
 export const TrainingTypeMap = {
   [TrainingModeEnum.chunk]: {
-    name: 'chunk'
+    label: 'core.dataset.training.type chunk'
   },
   [TrainingModeEnum.qa]: {
-    name: 'qa'
+    label: 'core.dataset.training.type qa'
+  }
+};
+
+/* ------------ search -------------- */
+export enum DatasetSearchModeEnum {
+  embedding = 'embedding',
+  embeddingReRank = 'embeddingReRank',
+  embFullTextReRank = 'embFullTextReRank'
+}
+
+export const DatasetSearchModeMap = {
+  [DatasetSearchModeEnum.embedding]: {
+    icon: 'core/dataset/modeEmbedding',
+    title: 'core.dataset.search.mode.embedding',
+    desc: 'core.dataset.search.mode.embedding desc',
+    value: DatasetSearchModeEnum.embedding
+  },
+  [DatasetSearchModeEnum.embeddingReRank]: {
+    icon: 'core/dataset/modeEmbeddingRerank',
+    title: 'core.dataset.search.mode.embeddingReRank',
+    desc: 'core.dataset.search.mode.embeddingReRank desc',
+    value: DatasetSearchModeEnum.embeddingReRank
+  },
+  [DatasetSearchModeEnum.embFullTextReRank]: {
+    icon: 'core/dataset/modeEmbFTRerank',
+    title: 'core.dataset.search.mode.embFullTextReRank',
+    desc: 'core.dataset.search.mode.embFullTextReRank desc',
+    value: DatasetSearchModeEnum.embFullTextReRank
   }
-  // [TrainingModeEnum.hypothetical]: {
-  //   name: 'hypothetical'
-  // },
-  // [TrainingModeEnum.summary]: {
-  //   name: 'summary'
-  // },
-  // [TrainingModeEnum.multipleIndex]: {
-  //   name: 'multipleIndex'
-  // }
 };
 
 export const FolderAvatarSrc = '/imgs/files/folder.svg';

packages/global/core/dataset/controller.d.ts

@@ -5,6 +5,7 @@ export type CreateDatasetDataProps = {
   tmbId: string;
   datasetId: string;
   collectionId: string;
+  chunkIndex?: number;
   q: string;
   a?: string;
   indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];

packages/global/core/dataset/type.d.ts

@@ -4,6 +4,7 @@ import { PushDatasetDataChunkProps } from './api';
 import {
   DatasetCollectionTypeEnum,
   DatasetDataIndexTypeEnum,
+  DatasetStatusEnum,
   DatasetTypeEnum,
   TrainingModeEnum
 } from './constant';
@@ -20,26 +21,31 @@ export type DatasetSchemaType = {
   name: string;
   vectorModel: string;
   agentModel: string;
-  tags: string[];
+  intro: string;
   type: `${DatasetTypeEnum}`;
+  status: `${DatasetStatusEnum}`;
   permission: `${PermissionTypeEnum}`;
+  websiteConfig?: {
+    url: string;
+    selector: string;
+  };
 };
 
 export type DatasetCollectionSchemaType = {
   _id: string;
-  userId: string;
   teamId: string;
   tmbId: string;
   datasetId: string;
   parentId?: string;
   name: string;
   type: `${DatasetCollectionTypeEnum}`;
+  createTime: Date;
   updateTime: Date;
-  metadata: {
-    fileId?: string;
-    rawLink?: string;
-    pgCollectionId?: string;
-  };
+  trainingType: `${TrainingModeEnum}`;
+  chunkSize: number;
+  fileId?: string;
+  rawLink?: string;
+  metadata?: Record<string, any>;
 };
 
 export type DatasetDataIndexItemType = {
@@ -57,6 +63,8 @@ export type DatasetDataSchemaType = {
   collectionId: string;
   datasetId: string;
   collectionId: string;
+  chunkIndex: number;
+  updateTime: Date;
   q: string; // large chunks or question
   a: string; // answer or custom content
   fullTextToken: string;
@@ -78,6 +86,7 @@ export type DatasetTrainingSchemaType = {
   prompt: string;
   q: string;
   a: string;
+  chunkIndex: number;
   indexes: Omit<DatasetDataIndexItemType, 'dataId'>[];
 };
 
@@ -89,6 +98,18 @@ export type DatasetDataWithCollectionType = Omit<DatasetDataSchemaType, 'collect
 };
 
 /* ================= dataset ===================== */
+export type DatasetListItemType = {
+  _id: string;
+  parentId: string;
+  avatar: string;
+  name: string;
+  intro: string;
+  type: `${DatasetTypeEnum}`;
+  isOwner: boolean;
+  canWrite: boolean;
+  permission: `${PermissionTypeEnum}`;
+  vectorModel: VectorModelItemType;
+};
 export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel'> & {
   vectorModel: VectorModelItemType;
   agentModel: LLMModelItemType;
@@ -101,6 +122,7 @@ export type DatasetCollectionItemType = CollectionWithDatasetType & {
   canWrite: boolean;
   sourceName: string;
   sourceId?: string;
+  file?: DatasetFileSchema;
 };
 
 /* ================= data ===================== */

packages/global/core/module/constants.ts

@@ -61,7 +61,8 @@ export enum ModuleInputKeyEnum {
   datasetSelectList = 'datasets',
   datasetSimilarity = 'similarity',
   datasetLimit = 'limit',
-  datasetStartReRank = 'rerank',
+  datasetSearchMode = 'searchMode',
+  datasetParamsModal = 'datasetParamsModal',
 
   // context extract
   contextExtractInput = 'content',
@@ -98,5 +99,6 @@ export enum ModuleOutputKeyEnum {
 
 export enum VariableInputEnum {
   input = 'input',
+  textarea = 'textarea',
   select = 'select'
 }

packages/global/core/module/node/constant.ts

@@ -16,6 +16,8 @@ export enum FlowNodeInputTypeEnum {
   selectChatModel = 'selectChatModel',
   // dataset special input
   selectDataset = 'selectDataset',
+  selectDatasetParamsModal = 'selectDatasetParamsModal',
+
   hidden = 'hidden'
 }
 

packages/global/core/module/template/system/classifyQuestion.ts

@@ -17,8 +17,11 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
   flowType: FlowNodeTypeEnum.classifyQuestion,
   avatar: '/imgs/module/cq.png',
   name: '问题分类',
-  intro:
-    '根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：\n类型1: 打招呼\n类型2: 关于 laf 通用问题\n类型3: 关于 laf 代码问题\n类型4: 其他问题',
+  intro: `根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：
+类型1: 打招呼
+类型2: 关于商品“使用”问题
+类型3: 关于商品“购买”问题
+类型4: 其他问题`,
   showStatus: true,
   inputs: [
     Input_Template_TFSwitch,
@@ -39,7 +42,8 @@ export const ClassifyQuestionModule: FlowModuleTemplateType = {
       label: '背景知识',
       description:
         '你可以添加一些特定内容的介绍，从而更好的识别用户的问题类型。这个内容通常是给模型介绍一个它不知道的内容。',
-      placeholder: '例如: \n1. Laf 是一个云函数开发平台……\n2. Sealos 是一个集群操作系统',
+      placeholder:
+        '例如: \n1. AIGC（人工智能生成内容）是指使用人工智能技术自动或半自动地生成数字内容，如文本、图像、音乐、视频等。\n2. AIGC技术包括但不限于自然语言处理、计算机视觉、机器学习和深度学习。这些技术可以创建新内容或修改现有内容，以满足特定的创意、教育、娱乐或信息需求。',
       showTargetInApp: true,
       showTargetInPlugin: true
     },

packages/global/core/module/template/system/datasetSearch.ts

@@ -12,6 +12,7 @@ import {
 } from '../../constants';
 import { Input_Template_TFSwitch, Input_Template_UserChatInput } from '../input';
 import { Output_Template_Finish } from '../output';
+import { DatasetSearchModeEnum } from '../../../dataset/constant';
 
 export const DatasetSearchModule: FlowModuleTemplateType = {
   id: FlowNodeTypeEnum.datasetSearchNode,
@@ -36,7 +37,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
     },
     {
       key: ModuleInputKeyEnum.datasetSimilarity,
-      type: FlowNodeInputTypeEnum.slider,
+      type: FlowNodeInputTypeEnum.hidden,
       label: '最低相关性',
       value: 0.4,
       valueType: ModuleDataTypeEnum.number,
@@ -52,7 +53,7 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
     },
     {
       key: ModuleInputKeyEnum.datasetLimit,
-      type: FlowNodeInputTypeEnum.slider,
+      type: FlowNodeInputTypeEnum.hidden,
       label: '单次搜索上限',
       description: '最多取 n 条记录作为本次问题引用',
       value: 5,
@@ -68,13 +69,20 @@ export const DatasetSearchModule: FlowModuleTemplateType = {
       showTargetInPlugin: false
     },
     {
-      key: ModuleInputKeyEnum.datasetStartReRank,
-      type: FlowNodeInputTypeEnum.switch,
-      label: '结果重排',
-      description: '将召回的结果进行进一步重排，可增加召回率',
-      plusField: true,
-      value: false,
-      valueType: ModuleDataTypeEnum.boolean,
+      key: ModuleInputKeyEnum.datasetSearchMode,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: 'core.dataset.search.Mode',
+      valueType: ModuleDataTypeEnum.string,
+      showTargetInApp: false,
+      showTargetInPlugin: false,
+      value: DatasetSearchModeEnum.embedding
+    },
+    {
+      key: ModuleInputKeyEnum.datasetParamsModal,
+      type: FlowNodeInputTypeEnum.selectDatasetParamsModal,
+      label: '',
+      connected: false,
+      valueType: ModuleDataTypeEnum.any,
       showTargetInApp: false,
       showTargetInPlugin: false
     },

packages/global/core/module/template/system/history.ts

@@ -18,10 +18,12 @@ export const HistoryModule: FlowModuleTemplateType = {
       key: ModuleInputKeyEnum.historyMaxAmount,
       type: FlowNodeInputTypeEnum.numberInput,
       label: '最长记录数',
+      description:
+        '该记录数不代表模型可接收这么多的历史记录，具体可接收多少历史记录，取决于模型的能力，通常建议不要超过20条。',
       value: 6,
       valueType: ModuleDataTypeEnum.number,
       min: 0,
-      max: 50,
+      max: 100,
       showTargetInApp: false,
       showTargetInPlugin: false
     },

packages/global/package.json

@@ -3,13 +3,16 @@
   "version": "1.0.0",
   "dependencies": {
     "axios": "^1.5.1",
-    "timezones-list": "^3.0.2",
+    "cheerio": "1.0.0-rc.12",
     "dayjs": "^1.11.7",
     "encoding": "^0.1.13",
+    "js-tiktoken": "^1.0.7",
+    "node-html-markdown": "^1.3.0",
     "openai": "^4.16.1",
-    "js-tiktoken": "^1.0.7"
+    "timezones-list": "^3.0.2"
   },
   "devDependencies": {
-    "@types/node": "^20.8.5"
+    "@types/node": "^20.8.5",
+    "@types/turndown": "^5.0.4"
   }
 }

packages/global/support/wallet/bill/tools.ts

@@ -1,6 +1,7 @@
 /* bill common  */
 import { PRICE_SCALE } from './constants';
-import { BillItemType, BillSchema } from './type';
+import { BillSourceEnum } from './constants';
+import { AuthUserTypeEnum } from '../../permission/constant';
 
 /**
  * dataset price / PRICE_SCALE = real price
@@ -8,3 +9,15 @@ import { BillItemType, BillSchema } from './type';
 export const formatPrice = (val = 0, multiple = 1) => {
   return Number(((val / PRICE_SCALE) * multiple).toFixed(10));
 };
+
+export const getBillSourceByAuthType = ({
+  shareId,
+  authType
+}: {
+  shareId?: string;
+  authType?: `${AuthUserTypeEnum}`;
+}) => {
+  if (shareId) return BillSourceEnum.shareLink;
+  if (authType === AuthUserTypeEnum.apikey) return BillSourceEnum.api;
+  return BillSourceEnum.fastgpt;
+};

packages/service/common/api/plusRequest.ts

@@ -101,18 +101,18 @@ export function request(url: string, data: any, config: ConfigType, method: Meth
  * @param {Object} config
  * @returns
  */
-export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+export function GET<T = undefined>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
   return request(url, params, config, 'GET');
 }
 
-export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function POST<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
   return request(url, data, config, 'POST');
 }
 
-export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function PUT<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
   return request(url, data, config, 'PUT');
 }
 
-export function DELETE<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function DELETE<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
   return request(url, data, config, 'DELETE');
 }

packages/service/common/file/gridfs/controller.ts

@@ -89,7 +89,7 @@ export async function delFileById({
   return true;
 }
 
-export async function getDownloadBuf({
+export async function getDownloadStream({
   bucketName,
   fileId
 }: {
@@ -98,14 +98,5 @@ export async function getDownloadBuf({
 }) {
   const bucket = getGridBucket(bucketName);
 
-  const stream = bucket.openDownloadStream(new Types.ObjectId(fileId));
-
-  const buf: Buffer = await new Promise((resolve, reject) => {
-    const buffers: Buffer[] = [];
-    stream.on('data', (data) => buffers.push(data));
-    stream.on('error', reject);
-    stream.on('end', () => resolve(Buffer.concat(buffers)));
-  });
-
-  return buf;
+  return bucket.openDownloadStream(new Types.ObjectId(fileId));
 }

packages/service/core/chat/utils.ts

@@ -3,6 +3,7 @@ import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants
 import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
 import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
+import axios from 'axios';
 
 /* slice chat context by tokens */
 export function ChatContextFilter({
@@ -81,11 +82,13 @@ export function ChatContextFilter({
             }
         ]
  */
-export function formatStr2ChatContent(str: string) {
+export async function formatStr2ChatContent(str: string) {
   const content: ChatCompletionContentPart[] = [];
   let lastIndex = 0;
   const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
 
+  const imgKey: 'image_url' = 'image_url';
+
   let match;
 
   while ((match = regex.exec(str)) !== null) {
@@ -115,7 +118,7 @@ export function formatStr2ChatContent(str: string) {
 
       content.push(
         ...jsonLines.map((item) => ({
-          type: 'image_url' as any,
+          type: imgKey,
           image_url: {
             url: item.src
           }
@@ -148,5 +151,18 @@ export function formatStr2ChatContent(str: string) {
   if (content.length === 1 && content[0].type === 'text') {
     return content[0].text;
   }
+
+  if (!content) return null;
+  // load img to base64
+  for await (const item of content) {
+    if (item.type === imgKey && item[imgKey]?.url) {
+      const response = await axios.get(item[imgKey].url, {
+        responseType: 'arraybuffer'
+      });
+      const base64 = Buffer.from(response.data).toString('base64');
+      item[imgKey].url = `data:${response.headers['content-type']};base64,${base64}`;
+    }
+  }
+
   return content ? content : null;
 }

packages/service/core/dataset/collection/controller.ts

@@ -0,0 +1,73 @@
+import {
+  DatasetCollectionTrainingModeEnum,
+  DatasetCollectionTypeEnum
+} from '@fastgpt/global/core/dataset/constant';
+import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
+import { MongoDatasetCollection } from './schema';
+
+export async function createOneCollection({
+  name,
+  parentId,
+  datasetId,
+  type,
+  trainingType = DatasetCollectionTrainingModeEnum.manual,
+  chunkSize = 0,
+  fileId,
+  rawLink,
+  teamId,
+  tmbId,
+  metadata = {}
+}: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
+  const { _id } = await MongoDatasetCollection.create({
+    name,
+    teamId,
+    tmbId,
+    datasetId,
+    parentId: parentId || null,
+    type,
+    trainingType,
+    chunkSize,
+    fileId,
+    rawLink,
+    metadata
+  });
+
+  // create default collection
+  if (type === DatasetCollectionTypeEnum.folder) {
+    await createDefaultCollection({
+      datasetId,
+      parentId: _id,
+      teamId,
+      tmbId
+    });
+  }
+
+  return _id;
+}
+
+// create default collection
+export function createDefaultCollection({
+  name = '手动录入',
+  datasetId,
+  parentId,
+  teamId,
+  tmbId
+}: {
+  name?: '手动录入' | '手动标注';
+  datasetId: string;
+  parentId?: string;
+  teamId: string;
+  tmbId: string;
+}) {
+  return MongoDatasetCollection.create({
+    name,
+    teamId,
+    tmbId,
+    datasetId,
+    parentId,
+    type: DatasetCollectionTypeEnum.virtual,
+    trainingType: DatasetCollectionTrainingModeEnum.manual,
+    chunkSize: 0,
+    updateTime: new Date('2099')
+  });
+}

packages/service/core/dataset/collection/schema.ts

@@ -1,7 +1,10 @@
 import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
-import { DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetCollectionTrainingTypeMap,
+  DatasetCollectionTypeMap
+} from '@fastgpt/global/core/dataset/constant';
 import { DatasetCollectionName } from '../schema';
 import {
   TeamCollectionName,
@@ -36,33 +39,42 @@ const DatasetCollectionSchema = new Schema({
     ref: DatasetCollectionName,
     required: true
   },
-  name: {
-    type: String,
-    required: true
-  },
   type: {
     type: String,
     enum: Object.keys(DatasetCollectionTypeMap),
     required: true
   },
+
+  name: {
+    type: String,
+    required: true
+  },
+  createTime: {
+    type: Date,
+    default: () => new Date()
+  },
   updateTime: {
     type: Date,
     default: () => new Date()
   },
+  trainingType: {
+    type: String,
+    enum: Object.keys(DatasetCollectionTrainingTypeMap),
+    required: true
+  },
+  chunkSize: {
+    type: Number,
+    required: true
+  },
+  fileId: {
+    type: Schema.Types.ObjectId,
+    ref: 'dataset.files'
+  },
+  rawLink: {
+    type: String
+  },
   metadata: {
-    type: {
-      fileId: {
-        type: Schema.Types.ObjectId,
-        ref: 'dataset.files'
-      },
-      rawLink: {
-        type: String
-      },
-      // 451 初始化
-      pgCollectionId: {
-        type: String
-      }
-    },
+    type: Object,
     default: {}
   }
 });

packages/service/core/dataset/data/controller.ts

@@ -0,0 +1,75 @@
+import { MongoDatasetData } from './schema';
+import { deletePgDataById } from './pg';
+import { MongoDatasetTraining } from '../training/schema';
+import { delFileById } from '../../../common/file/gridfs/controller';
+import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { MongoDatasetCollection } from '../collection/schema';
+import { delDatasetFiles } from '../file/controller';
+import { delay } from '@fastgpt/global/common/system/utils';
+
+/* delete all data by datasetIds */
+export async function delDatasetRelevantData({ datasetIds }: { datasetIds: string[] }) {
+  datasetIds = datasetIds.map((item) => String(item));
+
+  // delete training data(There could be a training mission)
+  await MongoDatasetTraining.deleteMany({
+    datasetId: { $in: datasetIds }
+  });
+
+  // delete related files
+  await Promise.all(datasetIds.map((id) => delDatasetFiles({ datasetId: id })));
+
+  await delay(1000);
+
+  // delete pg data
+  await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`);
+  // delete dataset.datas
+  await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } });
+
+  // delete collections
+  await MongoDatasetCollection.deleteMany({
+    datasetId: { $in: datasetIds }
+  });
+}
+/**
+ * delete all data by collectionIds
+ */
+export async function delCollectionRelevantData({
+  collectionIds,
+  fileIds
+}: {
+  collectionIds: string[];
+  fileIds: string[];
+}) {
+  collectionIds = collectionIds.map((item) => String(item));
+  const filterFileIds = fileIds.filter(Boolean);
+
+  // delete training data
+  await MongoDatasetTraining.deleteMany({
+    collectionId: { $in: collectionIds }
+  });
+
+  // delete file
+  await Promise.all(
+    filterFileIds.map((fileId) => {
+      return delFileById({
+        bucketName: BucketNameEnum.dataset,
+        fileId
+      });
+    })
+  );
+
+  await delay(1000);
+
+  // delete pg data
+  await deletePgDataById(`collection_id IN ('${collectionIds.join("','")}')`);
+  // delete dataset.datas
+  await MongoDatasetData.deleteMany({ collectionId: { $in: collectionIds } });
+}
+/**
+ * delete one data by mongoDataId
+ */
+export async function delDatasetDataByDataId(mongoDataId: string) {
+  await deletePgDataById(['data_id', mongoDataId]);
+  await MongoDatasetData.findByIdAndDelete(mongoDataId);
+}

packages/service/core/dataset/data/pg.ts

@@ -0,0 +1,28 @@
+import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
+import { delay } from '@fastgpt/global/common/system/utils';
+import { PgClient } from '../../../common/pg';
+
+export async function deletePgDataById(
+  where: ['id' | 'dataset_id' | 'collection_id' | 'data_id', string] | string
+) {
+  let retry = 2;
+  async function deleteData(): Promise<any> {
+    try {
+      await PgClient.delete(PgDatasetTableName, {
+        where: [where]
+      });
+    } catch (error) {
+      if (--retry < 0) {
+        return Promise.reject(error);
+      }
+      await delay(500);
+      return deleteData();
+    }
+  }
+
+  await deleteData();
+
+  return {
+    tokenLen: 0
+  };
+}

packages/service/core/dataset/data/schema.ts

@@ -45,7 +45,7 @@ const DatasetDataSchema = new Schema({
   },
   fullTextToken: {
     type: String,
-    required: true
+    default: ''
   },
   indexes: {
     type: [
@@ -70,6 +70,18 @@ const DatasetDataSchema = new Schema({
       }
     ],
     default: []
+  },
+  // metadata
+  updateTime: {
+    type: Date,
+    default: () => new Date()
+  },
+  chunkIndex: {
+    type: Number,
+    default: 0
+  },
+  inited: {
+    type: Boolean
   }
 });
 
@@ -79,7 +91,7 @@ try {
   DatasetDataSchema.index({ collectionId: 1 });
   // full text index
   DatasetDataSchema.index({ datasetId: 1, fullTextToken: 'text' });
-  DatasetDataSchema.index({ fullTextToken: 1 });
+  DatasetDataSchema.index({ inited: 1 });
 } catch (error) {
   console.log(error);
 }

packages/service/core/dataset/schema.ts

@@ -1,7 +1,11 @@
 import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
-import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetStatusEnum,
+  DatasetStatusMap,
+  DatasetTypeMap
+} from '@fastgpt/global/core/dataset/constant';
 import {
   TeamCollectionName,
   TeamMemberCollectionName
@@ -31,9 +35,16 @@ const DatasetSchema = new Schema({
     ref: TeamMemberCollectionName,
     required: true
   },
-  updateTime: {
-    type: Date,
-    default: () => new Date()
+  type: {
+    type: String,
+    enum: Object.keys(DatasetTypeMap),
+    required: true,
+    default: 'dataset'
+  },
+  status: {
+    type: String,
+    enum: Object.keys(DatasetStatusMap),
+    default: DatasetStatusEnum.active
   },
   avatar: {
     type: String,
@@ -43,6 +54,10 @@ const DatasetSchema = new Schema({
     type: String,
     required: true
   },
+  updateTime: {
+    type: Date,
+    default: () => new Date()
+  },
   vectorModel: {
     type: String,
     required: true,
@@ -53,24 +68,26 @@ const DatasetSchema = new Schema({
     required: true,
     default: 'gpt-3.5-turbo-16k'
   },
-  type: {
+  intro: {
     type: String,
-    enum: Object.keys(DatasetTypeMap),
-    required: true,
-    default: 'dataset'
-  },
-  tags: {
-    type: [String],
-    default: [],
-    set(val: string | string[]) {
-      if (Array.isArray(val)) return val;
-      return val.split(' ').filter((item) => item);
-    }
+    default: ''
   },
   permission: {
     type: String,
     enum: Object.keys(PermissionTypeMap),
     default: PermissionTypeEnum.private
+  },
+  websiteConfig: {
+    type: {
+      url: {
+        type: String,
+        required: true
+      },
+      selector: {
+        type: String,
+        default: 'body'
+      }
+    }
   }
 });
 

packages/service/core/dataset/training/schema.ts

@@ -75,6 +75,10 @@ const TrainingDataSchema = new Schema({
     type: String,
     default: ''
   },
+  chunkIndex: {
+    type: Number,
+    default: 0
+  },
   indexes: {
     type: [
       {

packages/service/support/permission/controller.ts

@@ -105,14 +105,13 @@ export async function parseHeaderCert({
     };
   }
   // root user
-  async function parseRootKey(rootKey?: string, userId = '') {
+  async function parseRootKey(rootKey?: string) {
     if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
       return Promise.reject(ERROR_ENUM.unAuthorization);
     }
-    return userId;
   }
 
-  const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers ||
+  const { cookie, token, apikey, rootkey, authorization } = (req.headers ||
     {}) as ReqHeaderAuthType;
 
   const { uid, teamId, tmbId, appId, openApiKey, authType } = await (async () => {
@@ -129,9 +128,10 @@ export async function parseHeaderCert({
       };
     }
     if (authRoot && rootkey) {
+      await parseRootKey(rootkey);
       // root user
       return {
-        uid: await parseRootKey(rootkey, userid),
+        uid: '',
         teamId: '',
         tmbId: '',
         appId: '',

pnpm-lock.yaml

@@ -1,4 +1,4 @@
-lockfileVersion: '6.0'
+lockfileVersion: '6.1'
 
 settings:
   autoInstallPeers: true
@@ -45,6 +45,9 @@ importers:
       axios:
         specifier: ^1.5.1
         version: registry.npmmirror.com/axios@1.5.1
+      cheerio:
+        specifier: 1.0.0-rc.12
+        version: registry.npmmirror.com/cheerio@1.0.0-rc.12
       dayjs:
         specifier: ^1.11.7
         version: registry.npmmirror.com/dayjs@1.11.10
@@ -54,6 +57,9 @@ importers:
       js-tiktoken:
         specifier: ^1.0.7
         version: registry.npmmirror.com/js-tiktoken@1.0.7
+      node-html-markdown:
+        specifier: ^1.3.0
+        version: registry.npmmirror.com/node-html-markdown@1.3.0
       openai:
         specifier: ^4.16.1
         version: registry.npmmirror.com/openai@4.16.1(encoding@0.1.13)
@@ -64,6 +70,9 @@ importers:
       '@types/node':
         specifier: ^20.8.5
         version: registry.npmmirror.com/@types/node@20.8.7
+      '@types/turndown':
+        specifier: ^5.0.4
+        version: registry.npmmirror.com/@types/turndown@5.0.4
 
   packages/service:
     dependencies:
@@ -161,9 +170,6 @@ importers:
       '@fastgpt/web':
         specifier: workspace:*
         version: link:../../packages/web
-      '@mozilla/readability':
-        specifier: ^0.4.4
-        version: registry.npmmirror.com/@mozilla/readability@0.4.4
       '@node-rs/jieba':
         specifier: ^1.7.2
         version: registry.npmmirror.com/@node-rs/jieba@1.7.2
@@ -209,9 +215,6 @@ importers:
       jschardet:
         specifier: ^3.0.0
         version: registry.npmmirror.com/jschardet@3.0.0
-      jsdom:
-        specifier: ^22.1.0
-        version: registry.npmmirror.com/jsdom@22.1.0
       jsonwebtoken:
         specifier: ^9.0.2
         version: registry.npmmirror.com/jsonwebtoken@9.0.2
@@ -300,9 +303,6 @@ importers:
       '@types/js-cookie':
         specifier: ^3.0.3
         version: registry.npmmirror.com/@types/js-cookie@3.0.5
-      '@types/jsdom':
-        specifier: ^21.1.1
-        version: registry.npmmirror.com/@types/jsdom@21.1.4
       '@types/jsonwebtoken':
         specifier: ^9.0.3
         version: registry.npmmirror.com/@types/jsonwebtoken@9.0.4
@@ -3575,13 +3575,6 @@ packages:
     dev: false
     optional: true
 
-  registry.npmmirror.com/@mozilla/readability@0.4.4:
-    resolution: {integrity: sha512-MCgZyANpJ6msfvVMi6+A0UAsvZj//4OHREYUB9f2087uXHVoU+H+SWhuihvb1beKpM323bReQPRio0WNk2+V6g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@mozilla/readability/-/readability-0.4.4.tgz}
-    name: '@mozilla/readability'
-    version: 0.4.4
-    engines: {node: '>=14.0.0'}
-    dev: false
-
   registry.npmmirror.com/@next/env@13.5.2:
     resolution: {integrity: sha512-dUseBIQVax+XtdJPzhwww4GetTjlkRSsXeQnisIJWBaHsnxYcN2RGzsPHi58D6qnkATjnhuAtQTJmR1hKYQQPg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@next/env/-/env-13.5.2.tgz}
     name: '@next/env'
@@ -4248,13 +4241,6 @@ packages:
       use-sync-external-store: registry.npmmirror.com/use-sync-external-store@1.2.0(react@18.2.0)
     dev: false
 
-  registry.npmmirror.com/@tootallnate/once@2.0.0:
-    resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@tootallnate/once/-/once-2.0.0.tgz}
-    name: '@tootallnate/once'
-    version: 2.0.0
-    engines: {node: '>= 10'}
-    dev: false
-
   registry.npmmirror.com/@trysound/sax@0.2.0:
     resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@trysound/sax/-/sax-0.2.0.tgz}
     name: '@trysound/sax'
@@ -4618,16 +4604,6 @@ packages:
     version: 3.0.5
     dev: true
 
-  registry.npmmirror.com/@types/jsdom@21.1.4:
-    resolution: {integrity: sha512-NzAMLEV0KQ4cBaDx3Ls8VfJUElyDUm1xrtYRmcMK0gF8L5xYbujFVaQlJ50yinQ/d47j2rEP1XUzkiYrw4YRFA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/jsdom/-/jsdom-21.1.4.tgz}
-    name: '@types/jsdom'
-    version: 21.1.4
-    dependencies:
-      '@types/node': registry.npmmirror.com/@types/node@20.8.7
-      '@types/tough-cookie': registry.npmmirror.com/@types/tough-cookie@4.0.4
-      parse5: registry.npmmirror.com/parse5@7.1.2
-    dev: true
-
   registry.npmmirror.com/@types/json5@0.0.29:
     resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/json5/-/json5-0.0.29.tgz}
     name: '@types/json5'
@@ -4826,12 +4802,6 @@ packages:
       '@types/node': registry.npmmirror.com/@types/node@20.8.7
     dev: true
 
-  registry.npmmirror.com/@types/tough-cookie@4.0.4:
-    resolution: {integrity: sha512-95Sfz4nvMAb0Nl9DTxN3j64adfwfbBPEYq14VN7zT5J5O2M9V6iZMIIQU1U+pJyl9agHYHNCqhCXgyEtIRRa5A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/tough-cookie/-/tough-cookie-4.0.4.tgz}
-    name: '@types/tough-cookie'
-    version: 4.0.4
-    dev: true
-
   registry.npmmirror.com/@types/triple-beam@1.3.4:
     resolution: {integrity: sha512-HlJjF3wxV4R2VQkFpKe0YqJLilYNgtRtsqqZtby7RkVsSs+i+vbyzjtUwpFEdUCKcrGzCiEJE7F/0mKjh0sunA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/triple-beam/-/triple-beam-1.3.4.tgz}
     name: '@types/triple-beam'
@@ -4846,6 +4816,12 @@ packages:
       '@types/node': registry.npmmirror.com/@types/node@20.8.7
     dev: true
 
+  registry.npmmirror.com/@types/turndown@5.0.4:
+    resolution: {integrity: sha512-28GI33lCCkU4SGH1GvjDhFgOVr+Tym4PXGBIU1buJUa6xQolniPArtUT+kv42RR2N9MsMLInkr904Aq+ESHBJg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/turndown/-/turndown-5.0.4.tgz}
+    name: '@types/turndown'
+    version: 5.0.4
+    dev: true
+
   registry.npmmirror.com/@types/unist@2.0.9:
     resolution: {integrity: sha512-zC0iXxAv1C1ERURduJueYzkzZ2zaGyc+P2c95hgkikHPr3z8EdUZOlgEQ5X0DRmwDZn+hekycQnoeiiRVrmilQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/unist/-/unist-2.0.9.tgz}
     name: '@types/unist'
@@ -4967,12 +4943,6 @@ packages:
       '@zag-js/dom-query': registry.npmmirror.com/@zag-js/dom-query@0.16.0
     dev: false
 
-  registry.npmmirror.com/abab@2.0.6:
-    resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abab/-/abab-2.0.6.tgz}
-    name: abab
-    version: 2.0.6
-    dev: false
-
   registry.npmmirror.com/abort-controller@3.0.0:
     resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abort-controller/-/abort-controller-3.0.0.tgz}
     name: abort-controller
@@ -5001,17 +4971,6 @@ packages:
     hasBin: true
     dev: true
 
-  registry.npmmirror.com/agent-base@6.0.2:
-    resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/agent-base/-/agent-base-6.0.2.tgz}
-    name: agent-base
-    version: 6.0.2
-    engines: {node: '>= 6.0.0'}
-    dependencies:
-      debug: registry.npmmirror.com/debug@4.3.4
-    transitivePeerDependencies:
-      - supports-color
-    dev: false
-
   registry.npmmirror.com/agentkeepalive@4.5.0:
     resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz}
     name: agentkeepalive
@@ -5435,7 +5394,6 @@ packages:
     resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz}
     name: boolbase
     version: 1.0.0
-    dev: true
 
   registry.npmmirror.com/brace-expansion@1.1.11:
     resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz}
@@ -5720,6 +5678,34 @@ packages:
       get-func-name: registry.npmmirror.com/get-func-name@2.0.2
     dev: true
 
+  registry.npmmirror.com/cheerio-select@2.1.0:
+    resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cheerio-select/-/cheerio-select-2.1.0.tgz}
+    name: cheerio-select
+    version: 2.1.0
+    dependencies:
+      boolbase: registry.npmmirror.com/boolbase@1.0.0
+      css-select: registry.npmmirror.com/css-select@5.1.0
+      css-what: registry.npmmirror.com/css-what@6.1.0
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      domutils: registry.npmmirror.com/domutils@3.1.0
+    dev: false
+
+  registry.npmmirror.com/cheerio@1.0.0-rc.12:
+    resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cheerio/-/cheerio-1.0.0-rc.12.tgz}
+    name: cheerio
+    version: 1.0.0-rc.12
+    engines: {node: '>= 6'}
+    dependencies:
+      cheerio-select: registry.npmmirror.com/cheerio-select@2.1.0
+      dom-serializer: registry.npmmirror.com/dom-serializer@2.0.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      domutils: registry.npmmirror.com/domutils@3.1.0
+      htmlparser2: registry.npmmirror.com/htmlparser2@8.0.2
+      parse5: registry.npmmirror.com/parse5@7.1.2
+      parse5-htmlparser2-tree-adapter: registry.npmmirror.com/parse5-htmlparser2-tree-adapter@7.0.0
+    dev: false
+
   registry.npmmirror.com/chokidar@3.5.3:
     resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz}
     name: chokidar
@@ -6108,6 +6094,18 @@ packages:
       nth-check: registry.npmmirror.com/nth-check@2.1.1
     dev: true
 
+  registry.npmmirror.com/css-select@5.1.0:
+    resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/css-select/-/css-select-5.1.0.tgz}
+    name: css-select
+    version: 5.1.0
+    dependencies:
+      boolbase: registry.npmmirror.com/boolbase@1.0.0
+      css-what: registry.npmmirror.com/css-what@6.1.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      domutils: registry.npmmirror.com/domutils@3.1.0
+      nth-check: registry.npmmirror.com/nth-check@2.1.1
+    dev: false
+
   registry.npmmirror.com/css-tree@1.1.3:
     resolution: {integrity: sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/css-tree/-/css-tree-1.1.3.tgz}
     name: css-tree
@@ -6123,7 +6121,6 @@ packages:
     name: css-what
     version: 6.1.0
     engines: {node: '>= 6'}
-    dev: true
 
   registry.npmmirror.com/csso@4.2.0:
     resolution: {integrity: sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/csso/-/csso-4.2.0.tgz}
@@ -6134,15 +6131,6 @@ packages:
       css-tree: registry.npmmirror.com/css-tree@1.1.3
     dev: true
 
-  registry.npmmirror.com/cssstyle@3.0.0:
-    resolution: {integrity: sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cssstyle/-/cssstyle-3.0.0.tgz}
-    name: cssstyle
-    version: 3.0.0
-    engines: {node: '>=14'}
-    dependencies:
-      rrweb-cssom: registry.npmmirror.com/rrweb-cssom@0.6.0
-    dev: false
-
   registry.npmmirror.com/csstype@3.1.2:
     resolution: {integrity: sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/csstype/-/csstype-3.1.2.tgz}
     name: csstype
@@ -6539,17 +6527,6 @@ packages:
     version: 1.0.8
     dev: true
 
-  registry.npmmirror.com/data-urls@4.0.0:
-    resolution: {integrity: sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/data-urls/-/data-urls-4.0.0.tgz}
-    name: data-urls
-    version: 4.0.0
-    engines: {node: '>=14'}
-    dependencies:
-      abab: registry.npmmirror.com/abab@2.0.6
-      whatwg-mimetype: registry.npmmirror.com/whatwg-mimetype@3.0.0
-      whatwg-url: registry.npmmirror.com/whatwg-url@12.0.1
-    dev: false
-
   registry.npmmirror.com/date-fns@2.30.0:
     resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/date-fns/-/date-fns-2.30.0.tgz}
     name: date-fns
@@ -6591,12 +6568,6 @@ packages:
     dependencies:
       ms: registry.npmmirror.com/ms@2.1.2
 
-  registry.npmmirror.com/decimal.js@10.4.3:
-    resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/decimal.js/-/decimal.js-10.4.3.tgz}
-    name: decimal.js
-    version: 10.4.3
-    dev: false
-
   registry.npmmirror.com/decode-named-character-reference@1.0.2:
     resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz}
     name: decode-named-character-reference
@@ -6769,6 +6740,16 @@ packages:
       entities: registry.npmmirror.com/entities@2.2.0
     dev: true
 
+  registry.npmmirror.com/dom-serializer@2.0.0:
+    resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dom-serializer/-/dom-serializer-2.0.0.tgz}
+    name: dom-serializer
+    version: 2.0.0
+    dependencies:
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      entities: registry.npmmirror.com/entities@4.5.0
+    dev: false
+
   registry.npmmirror.com/domain-browser@4.23.0:
     resolution: {integrity: sha512-ArzcM/II1wCCujdCNyQjXrAFwS4mrLh4C7DZWlaI8mdh7h3BfKdNd3bKXITfl2PT9FtfQqaGvhi1vPRQPimjGA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domain-browser/-/domain-browser-4.23.0.tgz}
     name: domain-browser
@@ -6780,16 +6761,6 @@ packages:
     resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz}
     name: domelementtype
     version: 2.3.0
-    dev: true
-
-  registry.npmmirror.com/domexception@4.0.0:
-    resolution: {integrity: sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domexception/-/domexception-4.0.0.tgz}
-    name: domexception
-    version: 4.0.0
-    engines: {node: '>=12'}
-    dependencies:
-      webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
-    dev: false
 
   registry.npmmirror.com/domhandler@4.3.1:
     resolution: {integrity: sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domhandler/-/domhandler-4.3.1.tgz}
@@ -6800,6 +6771,15 @@ packages:
       domelementtype: registry.npmmirror.com/domelementtype@2.3.0
     dev: true
 
+  registry.npmmirror.com/domhandler@5.0.3:
+    resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domhandler/-/domhandler-5.0.3.tgz}
+    name: domhandler
+    version: 5.0.3
+    engines: {node: '>= 4'}
+    dependencies:
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+    dev: false
+
   registry.npmmirror.com/dompurify@3.0.6:
     resolution: {integrity: sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dompurify/-/dompurify-3.0.6.tgz}
     name: dompurify
@@ -6816,6 +6796,16 @@ packages:
       domhandler: registry.npmmirror.com/domhandler@4.3.1
     dev: true
 
+  registry.npmmirror.com/domutils@3.1.0:
+    resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domutils/-/domutils-3.1.0.tgz}
+    name: domutils
+    version: 3.1.0
+    dependencies:
+      dom-serializer: registry.npmmirror.com/dom-serializer@2.0.0
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+    dev: false
+
   registry.npmmirror.com/downloadjs@1.4.7:
     resolution: {integrity: sha512-LN1gO7+u9xjU5oEScGFKvXhYf7Y/empUIIEAGBs1LzUq/rg5duiDrkuH5A2lQGd5jfMOb9X9usDa2oVXwJ0U/Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/downloadjs/-/downloadjs-1.4.7.tgz}
     name: downloadjs
@@ -8285,6 +8275,13 @@ packages:
       space-separated-tokens: registry.npmmirror.com/space-separated-tokens@2.0.2
     dev: false
 
+  registry.npmmirror.com/he@1.2.0:
+    resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/he/-/he-1.2.0.tgz}
+    name: he
+    version: 1.2.0
+    hasBin: true
+    dev: false
+
   registry.npmmirror.com/heap@0.2.7:
     resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/heap/-/heap-0.2.7.tgz}
     name: heap
@@ -8321,15 +8318,6 @@ packages:
     dependencies:
       react-is: registry.npmmirror.com/react-is@16.13.1
 
-  registry.npmmirror.com/html-encoding-sniffer@3.0.0:
-    resolution: {integrity: sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz}
-    name: html-encoding-sniffer
-    version: 3.0.0
-    engines: {node: '>=12'}
-    dependencies:
-      whatwg-encoding: registry.npmmirror.com/whatwg-encoding@2.0.0
-    dev: false
-
   registry.npmmirror.com/html-parse-stringify@3.0.1:
     resolution: {integrity: sha512-KknJ50kTInJ7qIScF3jeaFRpMpE8/lfiTdzf/twXyPBLAGrLRTmkz3AdTnKeh40X8k9L2fdYwEp/42WGXIRGcg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/html-parse-stringify/-/html-parse-stringify-3.0.1.tgz}
     name: html-parse-stringify
@@ -8337,17 +8325,15 @@ packages:
     dependencies:
       void-elements: registry.npmmirror.com/void-elements@3.1.0
 
-  registry.npmmirror.com/http-proxy-agent@5.0.0:
-    resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz}
-    name: http-proxy-agent
-    version: 5.0.0
-    engines: {node: '>= 6'}
+  registry.npmmirror.com/htmlparser2@8.0.2:
+    resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/htmlparser2/-/htmlparser2-8.0.2.tgz}
+    name: htmlparser2
+    version: 8.0.2
     dependencies:
-      '@tootallnate/once': registry.npmmirror.com/@tootallnate/once@2.0.0
-      agent-base: registry.npmmirror.com/agent-base@6.0.2
-      debug: registry.npmmirror.com/debug@4.3.4
-    transitivePeerDependencies:
-      - supports-color
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      domutils: registry.npmmirror.com/domutils@3.1.0
+      entities: registry.npmmirror.com/entities@4.5.0
     dev: false
 
   registry.npmmirror.com/https-browserify@1.0.0:
@@ -8356,18 +8342,6 @@ packages:
     version: 1.0.0
     dev: true
 
-  registry.npmmirror.com/https-proxy-agent@5.0.1:
-    resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz}
-    name: https-proxy-agent
-    version: 5.0.1
-    engines: {node: '>= 6'}
-    dependencies:
-      agent-base: registry.npmmirror.com/agent-base@6.0.2
-      debug: registry.npmmirror.com/debug@4.3.4
-    transitivePeerDependencies:
-      - supports-color
-    dev: false
-
   registry.npmmirror.com/human-signals@4.3.1:
     resolution: {integrity: sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/human-signals/-/human-signals-4.3.1.tgz}
     name: human-signals
@@ -8743,12 +8717,6 @@ packages:
     engines: {node: '>=12'}
     dev: false
 
-  registry.npmmirror.com/is-potential-custom-element-name@1.0.1:
-    resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz}
-    name: is-potential-custom-element-name
-    version: 1.0.1
-    dev: false
-
   registry.npmmirror.com/is-regex@1.1.4:
     resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-regex/-/is-regex-1.1.4.tgz}
     name: is-regex
@@ -8921,46 +8889,6 @@ packages:
     engines: {node: '>=0.1.90'}
     dev: false
 
-  registry.npmmirror.com/jsdom@22.1.0:
-    resolution: {integrity: sha512-/9AVW7xNbsBv6GfWho4TTNjEo9fe6Zhf9O7s0Fhhr3u+awPwAJMKwAMXnkk5vBxflqLW9hTHX/0cs+P3gW+cQw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsdom/-/jsdom-22.1.0.tgz}
-    name: jsdom
-    version: 22.1.0
-    engines: {node: '>=16'}
-    peerDependencies:
-      canvas: ^2.5.0
-    peerDependenciesMeta:
-      canvas:
-        optional: true
-    dependencies:
-      abab: registry.npmmirror.com/abab@2.0.6
-      cssstyle: registry.npmmirror.com/cssstyle@3.0.0
-      data-urls: registry.npmmirror.com/data-urls@4.0.0
-      decimal.js: registry.npmmirror.com/decimal.js@10.4.3
-      domexception: registry.npmmirror.com/domexception@4.0.0
-      form-data: registry.npmmirror.com/form-data@4.0.0
-      html-encoding-sniffer: registry.npmmirror.com/html-encoding-sniffer@3.0.0
-      http-proxy-agent: registry.npmmirror.com/http-proxy-agent@5.0.0
-      https-proxy-agent: registry.npmmirror.com/https-proxy-agent@5.0.1
-      is-potential-custom-element-name: registry.npmmirror.com/is-potential-custom-element-name@1.0.1
-      nwsapi: registry.npmmirror.com/nwsapi@2.2.7
-      parse5: registry.npmmirror.com/parse5@7.1.2
-      rrweb-cssom: registry.npmmirror.com/rrweb-cssom@0.6.0
-      saxes: registry.npmmirror.com/saxes@6.0.0
-      symbol-tree: registry.npmmirror.com/symbol-tree@3.2.4
-      tough-cookie: registry.npmmirror.com/tough-cookie@4.1.3
-      w3c-xmlserializer: registry.npmmirror.com/w3c-xmlserializer@4.0.0
-      webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
-      whatwg-encoding: registry.npmmirror.com/whatwg-encoding@2.0.0
-      whatwg-mimetype: registry.npmmirror.com/whatwg-mimetype@3.0.0
-      whatwg-url: registry.npmmirror.com/whatwg-url@12.0.1
-      ws: registry.npmmirror.com/ws@8.14.2
-      xml-name-validator: registry.npmmirror.com/xml-name-validator@4.0.0
-    transitivePeerDependencies:
-      - bufferutil
-      - supports-color
-      - utf-8-validate
-    dev: false
-
   registry.npmmirror.com/jsesc@0.5.0:
     resolution: {integrity: sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsesc/-/jsesc-0.5.0.tgz}
     name: jsesc
@@ -10343,6 +10271,24 @@ packages:
       whatwg-url: registry.npmmirror.com/whatwg-url@5.0.0
     dev: false
 
+  registry.npmmirror.com/node-html-markdown@1.3.0:
+    resolution: {integrity: sha512-OeFi3QwC/cPjvVKZ114tzzu+YoR+v9UXW5RwSXGUqGb0qCl0DvP406tzdL7SFn8pZrMyzXoisfG2zcuF9+zw4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-html-markdown/-/node-html-markdown-1.3.0.tgz}
+    name: node-html-markdown
+    version: 1.3.0
+    engines: {node: '>=10.0.0'}
+    dependencies:
+      node-html-parser: registry.npmmirror.com/node-html-parser@6.1.11
+    dev: false
+
+  registry.npmmirror.com/node-html-parser@6.1.11:
+    resolution: {integrity: sha512-FAgwwZ6h0DSDWxfD0Iq1tsDcBCxdJB1nXpLPPxX8YyVWzbfCjKWEzaynF4gZZ/8hziUmp7ZSaKylcn0iKhufUQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-html-parser/-/node-html-parser-6.1.11.tgz}
+    name: node-html-parser
+    version: 6.1.11
+    dependencies:
+      css-select: registry.npmmirror.com/css-select@5.1.0
+      he: registry.npmmirror.com/he@1.2.0
+    dev: false
+
   registry.npmmirror.com/node-releases@2.0.13:
     resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-releases/-/node-releases-2.0.13.tgz}
     name: node-releases
@@ -10416,13 +10362,6 @@ packages:
     version: 2.1.1
     dependencies:
       boolbase: registry.npmmirror.com/boolbase@1.0.0
-    dev: true
-
-  registry.npmmirror.com/nwsapi@2.2.7:
-    resolution: {integrity: sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/nwsapi/-/nwsapi-2.2.7.tgz}
-    name: nwsapi
-    version: 2.2.7
-    dev: false
 
   registry.npmmirror.com/object-assign@4.1.1:
     resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz}
@@ -10703,12 +10642,22 @@ packages:
       json-parse-even-better-errors: registry.npmmirror.com/json-parse-even-better-errors@2.3.1
       lines-and-columns: registry.npmmirror.com/lines-and-columns@1.2.4
 
+  registry.npmmirror.com/parse5-htmlparser2-tree-adapter@7.0.0:
+    resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz}
+    name: parse5-htmlparser2-tree-adapter
+    version: 7.0.0
+    dependencies:
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      parse5: registry.npmmirror.com/parse5@7.1.2
+    dev: false
+
   registry.npmmirror.com/parse5@7.1.2:
     resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/parse5/-/parse5-7.1.2.tgz}
     name: parse5
     version: 7.1.2
     dependencies:
       entities: registry.npmmirror.com/entities@4.5.0
+    dev: false
 
   registry.npmmirror.com/path-browserify@1.0.1:
     resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/path-browserify/-/path-browserify-1.0.1.tgz}
@@ -11060,12 +11009,6 @@ packages:
     version: 1.1.0
     dev: false
 
-  registry.npmmirror.com/psl@1.9.0:
-    resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/psl/-/psl-1.9.0.tgz}
-    name: psl
-    version: 1.9.0
-    dev: false
-
   registry.npmmirror.com/public-encrypt@4.0.3:
     resolution: {integrity: sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/public-encrypt/-/public-encrypt-4.0.3.tgz}
     name: public-encrypt
@@ -11106,12 +11049,6 @@ packages:
     engines: {node: '>=0.4.x'}
     dev: true
 
-  registry.npmmirror.com/querystringify@2.2.0:
-    resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/querystringify/-/querystringify-2.2.0.tgz}
-    name: querystringify
-    version: 2.2.0
-    dev: false
-
   registry.npmmirror.com/queue-microtask@1.2.3:
     resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/queue-microtask/-/queue-microtask-1.2.3.tgz}
     name: queue-microtask
@@ -11628,12 +11565,6 @@ packages:
     engines: {node: '>=4'}
     dev: false
 
-  registry.npmmirror.com/requires-port@1.0.0:
-    resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/requires-port/-/requires-port-1.0.0.tgz}
-    name: requires-port
-    version: 1.0.0
-    dev: false
-
   registry.npmmirror.com/resolve-from@4.0.0:
     resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/resolve-from/-/resolve-from-4.0.0.tgz}
     name: resolve-from
@@ -11724,12 +11655,6 @@ packages:
       fsevents: registry.npmmirror.com/fsevents@2.3.3
     dev: true
 
-  registry.npmmirror.com/rrweb-cssom@0.6.0:
-    resolution: {integrity: sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz}
-    name: rrweb-cssom
-    version: 0.6.0
-    dev: false
-
   registry.npmmirror.com/run-parallel@1.2.0:
     resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/run-parallel/-/run-parallel-1.2.0.tgz}
     name: run-parallel
@@ -11820,15 +11745,6 @@ packages:
       immutable: registry.npmmirror.com/immutable@4.3.4
       source-map-js: registry.npmmirror.com/source-map-js@1.0.2
 
-  registry.npmmirror.com/saxes@6.0.0:
-    resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/saxes/-/saxes-6.0.0.tgz}
-    name: saxes
-    version: 6.0.0
-    engines: {node: '>=v12.22.7'}
-    dependencies:
-      xmlchars: registry.npmmirror.com/xmlchars@2.2.0
-    dev: false
-
   registry.npmmirror.com/scheduler@0.23.0:
     resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/scheduler/-/scheduler-0.23.0.tgz}
     name: scheduler
@@ -12271,12 +12187,6 @@ packages:
       stable: registry.npmmirror.com/stable@0.1.8
     dev: true
 
-  registry.npmmirror.com/symbol-tree@3.2.4:
-    resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/symbol-tree/-/symbol-tree-3.2.4.tgz}
-    name: symbol-tree
-    version: 3.2.4
-    dev: false
-
   registry.npmmirror.com/tapable@2.2.1:
     resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tapable/-/tapable-2.2.1.tgz}
     name: tapable
@@ -12351,18 +12261,6 @@ packages:
     version: 1.0.6
     dev: false
 
-  registry.npmmirror.com/tough-cookie@4.1.3:
-    resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tough-cookie/-/tough-cookie-4.1.3.tgz}
-    name: tough-cookie
-    version: 4.1.3
-    engines: {node: '>=6'}
-    dependencies:
-      psl: registry.npmmirror.com/psl@1.9.0
-      punycode: registry.npmmirror.com/punycode@2.3.0
-      universalify: registry.npmmirror.com/universalify@0.2.0
-      url-parse: registry.npmmirror.com/url-parse@1.5.10
-    dev: false
-
   registry.npmmirror.com/tr46@0.0.3:
     resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-0.0.3.tgz}
     name: tr46
@@ -12378,15 +12276,6 @@ packages:
       punycode: registry.npmmirror.com/punycode@2.3.0
     dev: false
 
-  registry.npmmirror.com/tr46@4.1.1:
-    resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-4.1.1.tgz}
-    name: tr46
-    version: 4.1.1
-    engines: {node: '>=14'}
-    dependencies:
-      punycode: registry.npmmirror.com/punycode@2.3.0
-    dev: false
-
   registry.npmmirror.com/trim-lines@3.0.1:
     resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/trim-lines/-/trim-lines-3.0.1.tgz}
     name: trim-lines
@@ -12786,13 +12675,6 @@ packages:
       unist-util-visit-parents: registry.npmmirror.com/unist-util-visit-parents@5.1.3
     dev: false
 
-  registry.npmmirror.com/universalify@0.2.0:
-    resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/universalify/-/universalify-0.2.0.tgz}
-    name: universalify
-    version: 0.2.0
-    engines: {node: '>= 4.0.0'}
-    dev: false
-
   registry.npmmirror.com/update-browserslist-db@1.0.13(browserslist@4.22.1):
     resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz}
     id: registry.npmmirror.com/update-browserslist-db/1.0.13
@@ -12814,15 +12696,6 @@ packages:
       punycode: registry.npmmirror.com/punycode@2.3.0
     dev: true
 
-  registry.npmmirror.com/url-parse@1.5.10:
-    resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/url-parse/-/url-parse-1.5.10.tgz}
-    name: url-parse
-    version: 1.5.10
-    dependencies:
-      querystringify: registry.npmmirror.com/querystringify@2.2.0
-      requires-port: registry.npmmirror.com/requires-port@1.0.0
-    dev: false
-
   registry.npmmirror.com/url@0.11.3:
     resolution: {integrity: sha512-6hxOLGfZASQK/cijlZnZJTq8OXAkt/3YGfQX45vvMYXpZoo8NdWZcY73K108Jf759lS1Bv/8wXnHDTSz17dSRw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/url/-/url-0.11.3.tgz}
     name: url
@@ -13073,15 +12946,6 @@ packages:
     version: 3.1.0
     engines: {node: '>=0.10.0'}
 
-  registry.npmmirror.com/w3c-xmlserializer@4.0.0:
-    resolution: {integrity: sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz}
-    name: w3c-xmlserializer
-    version: 4.0.0
-    engines: {node: '>=14'}
-    dependencies:
-      xml-name-validator: registry.npmmirror.com/xml-name-validator@4.0.0
-    dev: false
-
   registry.npmmirror.com/watchpack@2.4.0:
     resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/watchpack/-/watchpack-2.4.0.tgz}
     name: watchpack
@@ -13130,22 +12994,6 @@ packages:
     engines: {node: '>=12'}
     dev: false
 
-  registry.npmmirror.com/whatwg-encoding@2.0.0:
-    resolution: {integrity: sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz}
-    name: whatwg-encoding
-    version: 2.0.0
-    engines: {node: '>=12'}
-    dependencies:
-      iconv-lite: registry.npmmirror.com/iconv-lite@0.6.3
-    dev: false
-
-  registry.npmmirror.com/whatwg-mimetype@3.0.0:
-    resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz}
-    name: whatwg-mimetype
-    version: 3.0.0
-    engines: {node: '>=12'}
-    dev: false
-
   registry.npmmirror.com/whatwg-url@11.0.0:
     resolution: {integrity: sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-11.0.0.tgz}
     name: whatwg-url
@@ -13156,16 +13004,6 @@ packages:
       webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
     dev: false
 
-  registry.npmmirror.com/whatwg-url@12.0.1:
-    resolution: {integrity: sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-12.0.1.tgz}
-    name: whatwg-url
-    version: 12.0.1
-    engines: {node: '>=14'}
-    dependencies:
-      tr46: registry.npmmirror.com/tr46@4.1.1
-      webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
-    dev: false
-
   registry.npmmirror.com/whatwg-url@5.0.0:
     resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-5.0.0.tgz}
     name: whatwg-url
@@ -13308,28 +13146,6 @@ packages:
     name: wrappy
     version: 1.0.2
 
-  registry.npmmirror.com/ws@8.14.2:
-    resolution: {integrity: sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/ws/-/ws-8.14.2.tgz}
-    name: ws
-    version: 8.14.2
-    engines: {node: '>=10.0.0'}
-    peerDependencies:
-      bufferutil: ^4.0.1
-      utf-8-validate: '>=5.0.2'
-    peerDependenciesMeta:
-      bufferutil:
-        optional: true
-      utf-8-validate:
-        optional: true
-    dev: false
-
-  registry.npmmirror.com/xml-name-validator@4.0.0:
-    resolution: {integrity: sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz}
-    name: xml-name-validator
-    version: 4.0.0
-    engines: {node: '>=12'}
-    dev: false
-
   registry.npmmirror.com/xmlbuilder@10.1.1:
     resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz}
     name: xmlbuilder
@@ -13337,12 +13153,6 @@ packages:
     engines: {node: '>=4.0'}
     dev: false
 
-  registry.npmmirror.com/xmlchars@2.2.0:
-    resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlchars/-/xmlchars-2.2.0.tgz}
-    name: xmlchars
-    version: 2.2.0
-    dev: false
-
   registry.npmmirror.com/xtend@4.0.2:
     resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz}
     name: xtend

projects/app/data/config.json

@@ -113,6 +113,7 @@
       "maxToken": 3000
     }
   ],
+  "ReRankModels": [],
   "AudioSpeechModels": [
     {
       "model": "tts-1",

projects/app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app",
-  "version": "4.6.2",
+  "version": "4.6.3",
   "private": false,
   "scripts": {
     "dev": "next dev",
@@ -19,7 +19,6 @@
     "@fastgpt/global": "workspace:*",
     "@fastgpt/service": "workspace:*",
     "@fastgpt/web": "workspace:*",
-    "@mozilla/readability": "^0.4.4",
     "@node-rs/jieba": "^1.7.2",
     "@tanstack/react-query": "^4.24.10",
     "@types/nprogress": "^0.2.0",
@@ -35,7 +34,6 @@
     "i18next": "^22.5.1",
     "immer": "^9.0.19",
     "jschardet": "^3.0.0",
-    "jsdom": "^22.1.0",
     "jsonwebtoken": "^9.0.2",
     "lodash": "^4.17.21",
     "mammoth": "^1.6.0",
@@ -67,7 +65,6 @@
     "@types/downloadjs": "^1.4.3",
     "@types/formidable": "^2.0.5",
     "@types/js-cookie": "^3.0.3",
-    "@types/jsdom": "^21.1.1",
     "@types/jsonwebtoken": "^9.0.3",
     "@types/lodash": "^4.14.191",
     "@types/multer": "^1.4.10",

projects/app/public/docs/versionIntro.md

@@ -1,13 +1,11 @@
 ### Fast GPT V4.6.2
 
-1. 新增 - 团队空间
-2. 新增 - 多路向量（多个向量映射一组数据）
-3. 新增 - tts语音
-4. 新增 - 语音输入
-5. 新增 - 增强召回方式，提高召回精度
-6. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈
-7. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
-8. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
-9. [使用文档](https://doc.fastgpt.in/docs/intro/)
-10. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
-11. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)
+1. 商业版新增 - web站点同步
+2. 新增 - 集合元数据记录
+3. 优化 - url 读取内容
+4. 优化 - 流读取文件，防止内存溢出
+5. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
+6. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
+7. [使用文档](https://doc.fastgpt.in/docs/intro/)
+8. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
+9. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)

projects/app/public/locales/en/common.json

@@ -3,7 +3,6 @@
   "Cancel": "No",
   "Confirm": "Yes",
   "Create New": "Create",
-  "Dataset": "Dataset",
   "Export": "Export",
   "Folder": "Folder",
   "Move": "Move",
@@ -129,6 +128,7 @@
     "Choose": "Choose",
     "Close": "Close",
     "Collect": "Collect",
+    "Config": "Config",
     "Confirm": "Confirm",
     "Confirm Create": "Create",
     "Confirm Move": "Move here",
@@ -138,7 +138,6 @@
     "Course": "",
     "Create Failed": "Create Failed",
     "Create Success": "Create Success",
-    "Create Virtual File Failed": "Create Virtual File Failed",
     "Custom Title": "Custom Title",
     "Delete": "Delete",
     "Delete Failed": "Delete Failed",
@@ -152,6 +151,7 @@
     "Filed is repeat": "Filed is repeated",
     "Filed is repeated": "",
     "Input": "Input",
+    "Intro": "Intro",
     "Last Step": "Last",
     "Loading": "Loading",
     "Max credit": "Credit",
@@ -170,6 +170,7 @@
     "Rename Success": "Rename Success",
     "Request Error": "Request Error",
     "Require Input": "Required",
+    "Save": "Save",
     "Save Failed": "Save Failed",
     "Save Success": "Save Success",
     "Search": "Search",
@@ -189,6 +190,14 @@
     "Update Time": "Update Time",
     "Upload File Failed": "Upload File Failed",
     "Username": "UserName",
+    "Website": "Website",
+    "choosable": "choosable",
+    "confirm": {
+      "Common Tip": "Operational Confirm"
+    },
+    "empty": {
+      "Common Tip": "No data"
+    },
     "error": {
       "unKnow": "There was an accident"
     },
@@ -206,21 +215,29 @@
     "input": {
       "Repeat Value": "Repeat Value"
     },
+    "link": {
+      "UnValid": "UnValid Link"
+    },
     "speech": {
       "error tip": "Speech Failed"
     }
   },
   "core": {
+    "Max Token": "MaxTokens",
     "ai": {
       "Model": "Model",
       "Prompt": "Prompt"
     },
     "app": {
+      "App params config": "App Config",
       "Next Step Guide": "Next step guide",
       "Question Guide Tip": "At the end of the conversation, three leading questions will be asked.",
+      "Save and preview": "Save",
       "Select TTS": "Select TTS",
+      "Simple Config Tip": "Only basic functions are included. For complex agent functions, use advanced orchestration.",
       "TTS": "Audio Speech",
       "TTS Tip": "After this function is enabled, the voice playback function can be used after each conversation. Use of this feature may incur additional charges.",
+      "Welcome Text": "Welcome Text",
       "create app": "Create App",
       "setting": "App Setting",
       "simple": {
@@ -233,12 +250,13 @@
         "Speech model": "Speech model",
         "Speech speed": "Speed",
         "Test Listen": "Test",
-        "Test Listen Text": "Hello, this is FastGPT, how can I help you?",
+        "Test Listen Text": "Hello, this is a voice test, if you can hear this sentence, it means that the voice playback function is normal",
         "Web": "Browser (free)"
       }
     },
     "chat": {
       "Audio Speech Error": "Audio Speech Error",
+      "Quote Amount": "Dataset Quote:{{amount}}",
       "Record": "Speech",
       "Restart": "Restart",
       "Select File": "Select file",
@@ -247,29 +265,105 @@
       "Speaking": "I'm listening...",
       "Stop Speak": "Stop Speak",
       "Type a message": "Input problem",
+      "quote": {
+        "Quote Tip": "Only the actual reference content is displayed here. If the data is updated, it will not be updated in real time",
+        "Read Quote": "Read Quote",
+        "Read Source": "Read Source"
+      },
       "tts": {
         "Stop Speech": "Stop"
       }
     },
     "dataset": {
+      "All Dataset": "All Dataset",
       "Choose Dataset": "Choose Dataset",
+      "Common Dataset": "Common Dataset",
+      "Common Dataset Desc": "Knowledge bases can be built by importing files, web links, or manual entry",
       "Create dataset": "Create Dataset",
       "Dataset": "Dataset",
+      "Dataset Type": "Dataset Type",
+      "Delete Website Tips": "Confirm to delete the website",
+      "Empty Dataset": "",
+      "Empty Dataset Tips": "There is no knowledge base yet, go create one!",
+      "Folder Dataset": "Folder",
       "Go Dataset": "To Dataset",
+      "Intro Placeholder": "This dataset has not yet been introduced~",
       "Quote Length": "Quote Length",
       "Read Dataset": "Read Dataset",
       "Search Top K": "Top K",
       "Set Empty Result Tip": ",Response empty text",
+      "Set Website Config": "Configuring Website",
       "Similarity": "Similarity",
+      "Sync Time": "Update Time",
+      "Virtual File": "Virtual File",
+      "Website Dataset": "Website Sync",
+      "Website Dataset Desc": "Web site synchronization allows you to build a knowledge base directly from a web link",
+      "collection": {
+        "Click top config website": "Config",
+        "Empty Tip": "The collection is empty",
+        "Website Create Success": "Created successfully, data is being synchronized",
+        "Website Empty Tip": "No associated website yet,",
+        "Website Link": "Website Link",
+        "Website Sync": "Website",
+        "metadata": {
+          "Chunk Size": "Chunk Size",
+          "Createtime": "Create Time",
+          "Read Metadata": "Read Metadata",
+          "Training Type": "Training Type",
+          "Updatetime": "Update Time",
+          "metadata": "Metadata",
+          "read source": "Read Source",
+          "source": "Source",
+          "source name": "Source Name",
+          "source size": "Source Size"
+        },
+        "status": {
+          "active": "Ready",
+          "syncing": "Syncing"
+        },
+        "training": {
+          "type chunk": "Chunk",
+          "type manual": "Manual",
+          "type qa": "QA"
+        }
+      },
       "data": {
         "Edit": "Edit Data",
         "data is deleted": "Data is deleted",
         "id": "Data ID"
       },
+      "file": "File",
+      "folder": "Folder",
       "import": {
+        "Fetch Error": "Get link failed",
+        "Fetch Url": "Url",
+        "Fetch url placeholder": "Up to 10 links, one per line.",
+        "Fetch url tip": "Only static links can be read, please check the results",
         "Ideal chunk length": "Ideal chunk length",
         "Ideal chunk length Tips": "Segment by end symbol. We recommend that your document should be properly punctuated to ensure that each complete sentence length does not exceed this value \n Chinese document recommended 400~1000\n English document recommended 600~1200"
       },
+      "link": "Link",
+      "search": {
+        "Empty result response": "Empty Response",
+        "Empty result response Tips": "If you fill in the content, if no suitable content is found, you will directly reply to the content.",
+        "Min Similarity": "Min Similarity",
+        "Min Similarity Tips": "The similarity of different index models is different, please use the search test to select the appropriate value",
+        "Params Setting": "Params Setting",
+        "Top K": "Top K",
+        "mode": {
+          "embFullTextReRank": "Hybrid search ",
+          "embFullTextReRank desc": "Reordering with a mixture of vector search and full-text search results by Rerank usually works best",
+          "embedding": "Vector search",
+          "embedding desc": "Direct vector topk correlation query ",
+          "embeddingReRank": "Enhanced semantic retrieval ",
+          "embeddingReRank desc": "Sort using Rerank after overperforming vector topk queries "
+        },
+        "search mode": "Search Mode"
+      },
+      "status": {
+        "active": "Ready",
+        "syncing": "Syncing"
+      },
       "test": {
         "Test": "Start",
         "Test Result": "Results",
@@ -279,6 +373,20 @@
         "test history": "Test History",
         "test result placeholder": "The test results will be presented here",
         "test result tip": "The contents of the knowledge base are sorted according to their similarity to the test text, and you can adjust the corresponding text according to the test results. Note: The data in the test record may have been modified, clicking on a test data will show the latest data."
+      },
+      "training": {
+        "Website Sync": "Website Sync",
+        "type chunk": "Chunk",
+        "type qa": "QA"
+      },
+      "website": {
+        "Base Url": "BaseUrl",
+        "Config": "Website Configuring",
+        "Config Description": "The Web site synchronization function allows you to fill in the root address of a website, and the system will automatically crawl the relevant pages deeply for knowledge base training. Only crawls static websites, mainly project documents and blogs.",
+        "Confirm Create Tips": "Confirm to synchronize the site, the synchronization task will start later, please confirm!",
+        "Confirm Update Tips": "Are you sure to update the site configuration? The synchronization starts immediately with the new configuration. Please confirm",
+        "Selector": "Selector",
+        "Start Sync": "Start Sync"
       }
     },
     "module": {
@@ -327,18 +435,20 @@
       },
       "variable": {
         "add option": "Add Option",
+        "input type": "Text",
         "key": "Key",
         "key is required": "variable key is required",
         "select type": "Select",
         "text max length": "Max Length",
-        "text type": "Text",
+        "textarea type": "Textarea",
         "variable key is required": "",
         "variable name": "Name",
         "variable name is required": "variable name is required",
         "variable option is required": "Variable option is required",
         "variable option is value is required": "Variable option is value is required",
         "variable options": "Options"
-      }
+      },
+      "variable add option": "Add Option"
     }
   },
   "dataset": {
@@ -382,7 +492,6 @@
       "Create Training Data": "Training-{{filename}}",
       "Create Virtual File Success": "Create Virtual File Success",
       "Data Amount": "Data Amount",
-      "Ready": "Ready",
       "Select Collection": "Select Collection",
       "Select One Collection To Store": "Select the collection to store"
     },
@@ -412,11 +521,12 @@
     "deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!",
     "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!",
     "import csv tip": "Ensure that the CSV is in UTF-8 format; otherwise, garbled characters will be displayed",
-    "recall": {
-      "rerank": "Rerank"
-    },
     "test": {
       "noResult": "Search results are empty"
+    },
+    "website": {
+      "Base Url": "BaseUrl",
+      "Selector": "Selector"
     }
   },
   "error": {
@@ -433,7 +543,6 @@
     "Fetch Url": "Fetch Url",
     "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format",
     "Parse": "{{name}} Parsing...",
-    "Ready": "Ready",
     "Release the mouse to upload the file": "Release the mouse to upload the file",
     "Select a maximum of 10 files": "Select a maximum of 10 files",
     "Uploading": "Uploading: {{name}}, Progress: {{percent}}%",
@@ -673,8 +782,13 @@
   "wallet": {
     "bill": {
       "Audio Speech": "Audio Speech",
+      "ReRank": "ReRank",
       "Whisper": "Whisper",
       "bill username": "User"
+    },
+    "moduleName": {
+      "index": "Index Generation",
+      "qa": "QA Generation"
     }
   }
 }

projects/app/public/locales/zh/common.json

@@ -3,7 +3,6 @@
   "Cancel": "取消",
   "Confirm": "确认",
   "Create New": "新建",
-  "Dataset": "知识库",
   "Export": "导出",
   "Folder": "文件夹",
   "Move": "移动",
@@ -129,6 +128,7 @@
     "Choose": "选择",
     "Close": "关闭",
     "Collect": "收藏",
+    "Config": "配置",
     "Confirm": "确认",
     "Confirm Create": "确认创建",
     "Confirm Move": "移动到这",
@@ -138,7 +138,6 @@
     "Course": "",
     "Create Failed": "创建异常",
     "Create Success": "创建成功",
-    "Create Virtual File Failed": "创建虚拟文件失败",
     "Custom Title": "自定义标题",
     "Delete": "删除",
     "Delete Failed": "删除失败",
@@ -152,6 +151,7 @@
     "Filed is repeat": "",
     "Filed is repeated": "字段重复了",
     "Input": "输入",
+    "Intro": "介绍",
     "Last Step": "上一步",
     "Loading": "加载中",
     "Max credit": "最大金额",
@@ -170,6 +170,7 @@
     "Rename Success": "重命名成功",
     "Request Error": "请求异常",
     "Require Input": "必填",
+    "Save": "保存",
     "Save Failed": "保存失败",
     "Save Success": "保存成功",
     "Search": "搜索",
@@ -189,6 +190,14 @@
     "Update Time": "更新时间",
     "Upload File Failed": "上传文件失败",
     "Username": "用户名",
+    "Website": "网站",
+    "choosable": "可选",
+    "confirm": {
+      "Common Tip": "操作确认"
+    },
+    "empty": {
+      "Common Tip": "没有什么数据噢~"
+    },
     "error": {
       "unKnow": "出现了点意外~"
     },
@@ -206,21 +215,29 @@
     "input": {
       "Repeat Value": "有重复的值"
     },
+    "link": {
+      "UnValid": "无效的链接"
+    },
     "speech": {
       "error tip": "语音转文字失败"
     }
   },
   "core": {
+    "Max Token": "单条数据上限",
     "ai": {
       "Model": "AI 模型",
       "Prompt": "提示词"
     },
     "app": {
+      "App params config": "应用配置",
       "Next Step Guide": "下一步指引",
       "Question Guide Tip": "对话结束后，会为生成 3 个引导性问题。",
+      "Save and preview": "保存并预览",
       "Select TTS": "选择语音播放模式",
+      "Simple Config Tip": "仅包含基础功能，复杂 agent 功能请使用高级编排。",
       "TTS": "语音播报",
       "TTS Tip": "开启后，每次对话后可使用语音播放功能。使用该功能可能产生额外费用。",
+      "Welcome Text": "对话开场白",
       "create app": "创建属于你的 AI 应用",
       "setting": "应用信息设置",
       "simple": {
@@ -233,12 +250,13 @@
         "Speech model": "语音模型",
         "Speech speed": "语速",
         "Test Listen": "试听",
-        "Test Listen Text": "你好，我是 FastGPT，有什么可以帮助你么？",
+        "Test Listen Text": "你好，这是语音测试，如果你能听到这句话，说明语音播放功能正常",
         "Web": "浏览器自带(免费)"
       }
     },
     "chat": {
       "Audio Speech Error": "语音播报异常",
+      "Quote Amount": "知识库引用({{amount}}条)",
       "Record": "语音输入",
       "Restart": "重开对话",
       "Select File": "选择文件",
@@ -247,29 +265,105 @@
       "Speaking": "我在听，请说...",
       "Stop Speak": "停止录音",
       "Type a message": "输入问题",
+      "quote": {
+        "Quote Tip": "此处仅显示实际引用内容，若数据有更新，此处不会实时更新",
+        "Read Quote": "查看引用",
+        "Read Source": "查看来源"
+      },
       "tts": {
         "Stop Speech": "停止"
       }
     },
     "dataset": {
+      "All Dataset": "全部知识库",
       "Choose Dataset": "关联知识库",
+      "Common Dataset": "通用知识库",
+      "Common Dataset Desc": "可通过导入文件、网页链接或手动录入形式构建知识库",
       "Create dataset": "创建一个知识库",
       "Dataset": "知识库",
+      "Dataset Type": "知识库类型",
+      "Delete Website Tips": "确认删除该站点？",
+      "Empty Dataset": "",
+      "Empty Dataset Tips": "还没有知识库，快去创建一个吧！",
+      "Folder Dataset": "文件夹",
       "Go Dataset": "前往知识库",
+      "Intro Placeholder": "这个知识库还没有介绍~",
       "Quote Length": "引用内容长度",
       "Read Dataset": "查看知识库详情",
       "Search Top K": "单次搜索数量",
       "Set Empty Result Tip": ",未搜索到内容时回复指定内容",
+      "Set Website Config": "开始配置网站信息",
       "Similarity": "相似度",
+      "Sync Time": "最后更新时间",
+      "Virtual File": "虚拟文件",
+      "Website Dataset": "Web 站点同步",
+      "Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库",
+      "collection": {
+        "Click top config website": "点击配置网站",
+        "Empty Tip": "数据集空空如也",
+        "Website Create Success": "创建成功，正在同步数据",
+        "Website Empty Tip": "还没有关联网站，",
+        "Website Link": "Web 站点地址",
+        "Website Sync": "Web 站点同步",
+        "metadata": {
+          "Chunk Size": "分割大小",
+          "Createtime": "创建时间",
+          "Read Metadata": "查看元数据",
+          "Training Type": "训练模式",
+          "Updatetime": "更新时间",
+          "metadata": "元数据",
+          "read source": "查看原始内容",
+          "source": "数据来源",
+          "source name": "来源名",
+          "source size": "来源大小"
+        },
+        "status": {
+          "active": "已就绪",
+          "syncing": "同步中"
+        },
+        "training": {
+          "type chunk": "直接分段",
+          "type manual": "手动",
+          "type qa": "问答拆分"
+        }
+      },
       "data": {
         "Edit": "编辑数据",
         "data is deleted": "该数据已被删除",
         "id": "数据ID"
       },
+      "file": "文件",
+      "folder": "目录",
       "import": {
+        "Fetch Error": "获取链接失败",
+        "Fetch Url": "网络链接",
+        "Fetch url placeholder": "最多10个链接，每行一个。",
+        "Fetch url tip": "仅支持读取静态链接，请注意检查结果",
         "Ideal chunk length": "理想分块长度",
         "Ideal chunk length Tips": "按结束符号进行分段。我们建议您的文档应合理的使用标点符号，以确保每个完整的句子长度不要超过该值\n中文文档建议400~1000\n英文文档建议600~1200"
       },
+      "link": "链接",
+      "search": {
+        "Empty result response": "空搜索回复",
+        "Empty result response Tips": "若填写该内容，没有搜索到合适内容时，将直接回复填写的内容。",
+        "Min Similarity": "最低相似度",
+        "Min Similarity Tips": "不同索引模型的相似度有区别，请通过搜索测试来选择合适的数值",
+        "Params Setting": "搜索参数设置",
+        "Top K": "单次搜索上限",
+        "mode": {
+          "embFullTextReRank": "混合检索",
+          "embFullTextReRank desc": "使用向量检索与全文检索混合结果进行 Rerank 进行重排，通常效果最佳",
+          "embedding": "语义检索",
+          "embedding desc": "直接进行向量 topk 相关性查询",
+          "embeddingReRank": "增强语义检索",
+          "embeddingReRank desc": "超额进行向量 topk 查询后再使用 Rerank 进行排序"
+        },
+        "search mode": "检索模式"
+      },
+      "status": {
+        "active": "已就绪",
+        "syncing": "同步中"
+      },
       "test": {
         "Test": "测试",
         "Test Result": "测试结果",
@@ -279,6 +373,20 @@
         "test history": "测试历史",
         "test result placeholder": "测试结果将在这里展示",
         "test result tip": "根据知识库内容与测试文本的相似度进行排序，你可以根据测试结果调整对应的文本。\n注意：测试记录中的数据可能已经被修改过，点击某条测试数据后将展示最新的数据。"
+      },
+      "training": {
+        "Website Sync": "Web 站点同步",
+        "type chunk": "直接分段",
+        "type qa": "问答拆分"
+      },
+      "website": {
+        "Base Url": "根地址",
+        "Config": "Web站点配置",
+        "Config Description": "Web 站点同步功能允许你填写一个网站的根地址，系统会自动深度抓取相关的网页进行知识库训练。仅会抓取静态的网站，以项目文档、博客为主。",
+        "Confirm Create Tips": "确认同步该站点，同步任务将随后开启，请确认！",
+        "Confirm Update Tips": "确认更新站点配置？会立即按新的配置开始同步，请确认！",
+        "Selector": "选择器",
+        "Start Sync": "开始同步"
       }
     },
     "module": {
@@ -327,18 +435,20 @@
       },
       "variable": {
         "add option": "添加选项",
+        "input type": "文本",
         "key": "变量 key",
-        "key is required": "",
+        "key is required": "变量key是必须的",
         "select type": "下拉单选",
         "text max length": "最大长度",
-        "text type": "文本",
+        "textarea type": "段落",
         "variable key is required": "变量 key 不能为空",
         "variable name": "变量名",
         "variable name is required": "变量名不能为空",
         "variable option is required": "选项不能全空",
         "variable option is value is required": "选项内容不能为空",
         "variable options": "选项"
-      }
+      },
+      "variable add option": "添加选项"
     }
   },
   "dataset": {
@@ -382,7 +492,6 @@
       "Create Training Data": "文件训练-{{filename}}",
       "Create Virtual File Success": "创建虚拟文件成功",
       "Data Amount": "数据总量",
-      "Ready": "已就绪",
       "Select Collection": "选择文件",
       "Select One Collection To Store": "选择一个文件进行存储"
     },
@@ -412,11 +521,12 @@
     "deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！",
     "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！",
     "import csv tip": "请确保CSV为UTF-8格式，否则会乱码",
-    "recall": {
-      "rerank": "结果重排"
-    },
     "test": {
       "noResult": "搜索结果为空"
+    },
+    "website": {
+      "Base Url": "",
+      "Selector": ""
     }
   },
   "error": {
@@ -433,7 +543,6 @@
     "Fetch Url": "链接读取",
     "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式",
     "Parse": "{{name}} 解析中...",
-    "Ready": "",
     "Release the mouse to upload the file": "松开鼠标上传文件",
     "Select a maximum of 10 files": "最多选择10个文件",
     "Uploading": "正在上传 {{name}}，进度: {{percent}}%",
@@ -673,8 +782,13 @@
   "wallet": {
     "bill": {
       "Audio Speech": "语音播报",
+      "ReRank": "结果重排",
       "Whisper": "语音输入",
       "bill username": "用户"
+    },
+    "moduleName": {
+      "index": "索引生成",
+      "qa": "QA 拆分"
     }
   }
 }

projects/app/public/simpleTemplates/fastgpt-simple.json

@@ -14,7 +14,7 @@
       "datasets": true,
       "similarity": false,
       "limit": false,
-      "rerank": false,
+      "searchMode": "embedding",
       "searchEmptyText": false
     },
     "userGuide": {

projects/app/src/components/ChatBox/MessageInput.tsx

@@ -74,8 +74,8 @@ const MessageInput = ({
       try {
         const src = await compressImgFileAndUpload({
           file: file.rawFile,
-          maxW: 1000,
-          maxH: 1000,
+          maxW: 4329,
+          maxH: 4329,
           maxSize: 1024 * 1024 * 5,
           // 30 day expired.
           expiredTime: addDays(new Date(), 30)
@@ -92,6 +92,8 @@ const MessageInput = ({
         );
       } catch (error) {
         setFileList((state) => state.filter((item) => item.id !== file.id));
+        console.log(error);
+
         toast({
           status: 'error',
           title: t('common.Upload File Failed')

projects/app/src/components/ChatBox/QuoteModal.tsx

@@ -71,9 +71,9 @@ const QuoteModal = ({
         iconSrc="/imgs/modal/quote.svg"
         title={
           <Box>
-            知识库引用({rawSearch.length}条)
-            <Box fontSize={'10px'} color={'myGray.500'} fontWeight={'normal'}>
-              注意: 修改知识库内容成功后，此处不会显示变更情况。点击编辑后，会显示知识库最新的内容。
+            {t('core.chat.Quote Amount', { amount: rawSearch.length })}
+            <Box fontSize={'sm'} color={'myGray.500'} fontWeight={'normal'}>
+              {t('core.chat.quote.Quote Tip')}
             </Box>
           </Box>
         }
@@ -122,7 +122,7 @@ const QuoteModal = ({
                 <Flex alignItems={'center'} fontSize={'sm'} mt={3} gap={4} color={'myGray.500'}>
                   {isPc && (
                     <MyTooltip label={t('core.dataset.data.id')}>
-                      <Flex border={theme.borders.base} px={3} borderRadius={'md'}>
+                      <Flex border={theme.borders.base} py={'1px'} px={3} borderRadius={'3px'}>
                         # {item.id}
                       </Flex>
                     </MyTooltip>

projects/app/src/components/ChatBox/ResponseTags.tsx

@@ -1,7 +1,7 @@
 import React, { useMemo, useState } from 'react';
 import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/api.d';
 import type { ChatItemType } from '@fastgpt/global/core/chat/type';
-import { Flex, BoxProps, useDisclosure, Image, useTheme } from '@chakra-ui/react';
+import { Flex, BoxProps, useDisclosure, Image, useTheme, Box } from '@chakra-ui/react';
 import { useTranslation } from 'next-i18next';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
@@ -11,6 +11,8 @@ import MyTooltip from '../MyTooltip';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
 import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
 import ChatBoxDivider from '@/components/core/chat/Divider';
+import MyIcon from '../Icon';
+import { getFileAndOpen } from '@/web/core/dataset/utils';
 
 const QuoteModal = dynamic(() => import('./QuoteModal'), { ssr: false });
 const ContextModal = dynamic(() => import('./ContextModal'), { ssr: false });
@@ -59,6 +61,7 @@ const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemRes
         .flat()
         .map((item) => ({
           sourceName: item.sourceName,
+          sourceId: item.sourceId,
           icon: getSourceNameIcon({ sourceId: item.sourceId, sourceName: item.sourceName })
         })),
       historyPreview: chatData?.historyPreview,
@@ -83,18 +86,67 @@ const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemRes
                 alignItems={'center'}
                 flexWrap={'wrap'}
                 fontSize={'sm'}
-                cursor={'pointer'}
                 border={theme.borders.sm}
                 py={1}
                 px={2}
                 borderRadius={'md'}
                 _hover={{
-                  bg: 'myBlue.100'
+                  '.controller': {
+                    display: 'flex'
+                  }
                 }}
+                overflow={'hidden'}
+                position={'relative'}
                 onClick={() => setQuoteModalData(quoteList)}
               >
                 <Image src={item.icon} alt={''} mr={1} w={'12px'} />
-                {item.sourceName}
+                <Box className="textEllipsis" flex={'1 0 0'}>
+                  {item.sourceName}
+                </Box>
+
+                <Box
+                  className="controller"
+                  display={['flex', 'none']}
+                  pr={2}
+                  position={'absolute'}
+                  right={0}
+                  left={0}
+                  justifyContent={'flex-end'}
+                  alignItems={'center'}
+                  h={'100%'}
+                  lineHeight={0}
+                  bg={`linear-gradient(to left, white,white ${
+                    item.sourceId ? '60px' : '30px'
+                  }, rgba(255,255,255,0) 80%)`}
+                >
+                  <MyTooltip label={t('core.chat.quote.Read Quote')}>
+                    <MyIcon
+                      name="common/viewLight"
+                      w={'14px'}
+                      cursor={'pointer'}
+                      _hover={{
+                        color: 'green.600'
+                      }}
+                    />
+                  </MyTooltip>
+                  {item.sourceId && (
+                    <MyTooltip label={t('core.chat.quote.Read Source')}>
+                      <MyIcon
+                        ml={4}
+                        name="common/routePushLight"
+                        w={'14px'}
+                        cursor={'pointer'}
+                        _hover={{ color: 'myBlue.600' }}
+                        onClick={async (e) => {
+                          e.stopPropagation();
+
+                          if (!item.sourceId) return;
+                          await getFileAndOpen(item.sourceId);
+                        }}
+                      />
+                    </MyTooltip>
+                  )}
+                </Box>
               </Flex>
             ))}
           </Flex>

projects/app/src/components/ChatBox/WholeResponseModal.tsx

@@ -10,6 +10,7 @@ import MyTooltip from '../MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
 import Markdown from '../Markdown';
+import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constant';
 
 function Row({ label, value }: { label: string; value?: string | number }) {
   const theme = useTheme();
@@ -116,7 +117,7 @@ const WholeResponseModal = ({
               if (!activeModule?.historyPreview) return '';
               return activeModule.historyPreview
                 .map((item, i) => `**${item.obj}**\n${item.value}`)
-                .join('\n---\n');
+                .join('\n\n---\n\n');
             })()}
           />
           {activeModule.quoteList && activeModule.quoteList.length > 0 && (
@@ -127,6 +128,13 @@ const WholeResponseModal = ({
           )}
 
           {/* dataset search */}
+          {activeModule?.searchMode && (
+            <Row
+              label={t('core.dataset.search.search mode')}
+              // @ts-ignore
+              value={t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
+            />
+          )}
           <Row label={t('chat.response.module similarity')} value={activeModule?.similarity} />
           <Row label={t('chat.response.module limit')} value={activeModule?.limit} />
 

projects/app/src/components/ChatBox/index.module.scss

@@ -1,14 +1,6 @@
 .stopIcon {
   animation: zoomStopIcon 0.4s infinite alternate;
 }
-@keyframes zoomStopIcon {
-  0% {
-    transform: scale(0.8);
-  }
-  100% {
-    transform: scale(1.2);
-  }
-}
 
 .statusAnimation {
   animation: statusBox 0.8s linear infinite alternate;

projects/app/src/components/ChatBox/index.tsx

@@ -26,7 +26,8 @@ import {
   useTheme,
   BoxProps,
   FlexProps,
-  Image
+  Image,
+  Textarea
 } from '@chakra-ui/react';
 import { feConfigs } from '@/web/common/system/staticData';
 import { eventBus } from '@/web/common/utils/eventbus';
@@ -558,11 +559,23 @@ const ChatBox = (
                       {item.type === VariableInputEnum.input && (
                         <Input
                           isDisabled={variableIsFinish}
+                          bg={'myWhite.400'}
                           {...register(item.key, {
                             required: item.required
                           })}
                         />
                       )}
+                      {item.type === VariableInputEnum.textarea && (
+                        <Textarea
+                          isDisabled={variableIsFinish}
+                          bg={'myWhite.400'}
+                          {...register(item.key, {
+                            required: item.required
+                          })}
+                          rows={5}
+                          maxLength={4000}
+                        />
+                      )}
                       {item.type === VariableInputEnum.select && (
                         <MySelect
                           width={'100%'}

projects/app/src/components/CommunityModal/index.tsx

@@ -3,12 +3,7 @@ import { Button, ModalFooter, ModalBody } from '@chakra-ui/react';
 import MyModal from '../MyModal';
 import { useTranslation } from 'next-i18next';
 import Markdown from '../Markdown';
-
-const md = `
-| 交流群 | 小助手 |
-| ----------------------- | -------------------- |
-| ![](https://otnvvf-imgs.oss.laf.run/wxqun300.jpg) | ![](https://otnvvf-imgs.oss.laf.run/wx300.jpg) |
-`;
+import { feConfigs } from '@/web/common/system/staticData';
 
 const CommunityModal = ({ onClose }: { onClose: () => void }) => {
   const { t } = useTranslation();
@@ -20,7 +15,7 @@ const CommunityModal = ({ onClose }: { onClose: () => void }) => {
       title={t('home.Community')}
     >
       <ModalBody textAlign={'center'}>
-        <Markdown source={md} />
+        <Markdown source={feConfigs?.concatMd || ''} />
       </ModalBody>
 
       <ModalFooter>

projects/app/src/components/EmptyTip/index.tsx

@@ -1,17 +1,19 @@
 import React from 'react';
 import { Flex, Box, FlexProps } from '@chakra-ui/react';
 import MyIcon from '../Icon';
+import { useTranslation } from 'next-i18next';
 
 type Props = FlexProps & {
-  text?: string | null;
+  text?: string | React.ReactNode;
 };
 
 const EmptyTip = ({ text, ...props }: Props) => {
+  const { t } = useTranslation();
   return (
     <Flex mt={5} flexDirection={'column'} alignItems={'center'} pt={'10vh'} {...props}>
       <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
       <Box mt={2} color={'myGray.500'}>
-        {text || '没有什么数据噢~'}
+        {text || t('common.empty.Common Tip')}
       </Box>
     </Flex>
   );

projects/app/src/components/Icon/icons/common/confirm/commonTip.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701410305116" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="8647" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M138.9056 180.5312m94.7712 0l563.3536 0q94.7712 0 94.7712 94.7712l0 576q0 94.7712-94.7712 94.7712l-563.3536 0q-94.7712 0-94.7712-94.7712l0-576q0-94.7712 94.7712-94.7712Z" fill="#CCDAFF" p-id="8648"></path><path d="M292.3008 81.92m64.768 0l305.3056 0q64.768 0 64.768 64.768l0 70.4q0 64.768-64.768 64.768l-305.3056 0q-64.768 0-64.768-64.768l0-70.4q0-64.768 64.768-64.768Z" fill="#7A7AF9" p-id="8649"></path><path d="M480.768 736.8704a49.408 49.408 0 0 1-34.7648-14.2848L332.8 611.3792a49.6128 49.6128 0 1 1 69.7344-70.7072l77.7216 76.4928L634.5216 460.8a49.6128 49.6128 0 1 1 70.6048 69.6832L516.096 721.92a49.6128 49.6128 0 0 1-35.328 14.9504z" fill="#7A7AF9" p-id="8650"></path></svg>

projects/app/src/components/Icon/icons/common/confirm/deleteTip.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701403554068" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5098" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M572.074667 337.134933a57.275733 57.275733 0 1 0-114.176-6.007466h-0.170667l12.936533 272.896v0.443733a44.544 44.544 0 1 0 89.019734-1.194667l12.424533-266.1376z m-196.949334-191.214933c76.151467-130.048 199.816533-129.774933 275.797334 0l340.3776 581.085867c76.117333 130.048 15.7696 235.4176-135.236267 235.4176H169.984c-150.8352 0-211.217067-105.6768-135.202133-235.4176L375.125333 145.92z m140.049067 687.581867a57.275733 57.275733 0 1 0 0-114.517334 57.275733 57.275733 0 0 0 0 114.517334z" fill="#FB6547" p-id="5099"></path></svg>

projects/app/src/components/Icon/icons/common/routePushLight.svg

@@ -0,0 +1,11 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701418384907"
+    class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="11367"
+    xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128">
+    <path
+        d="M892 928.1H134c-19.9 0-36-16.1-36-36v-758c0-19.9 16.1-36 36-36h314.1c19.9 0 36 16.1 36 36s-16.1 36-36 36H170v686h686V579.6c0-19.9 16.1-36 36-36s36 16.1 36 36v312.5c0 19.9-16.1 36-36 36z"
+        p-id="11368"></path>
+    <path
+        d="M927.9 131.6v-0.5c-0.1-1.7-0.4-3.3-0.7-4.9 0-0.1 0-0.2-0.1-0.3-0.4-1.7-0.9-3.3-1.5-4.9v-0.1c-0.6-1.6-1.4-3.1-2.2-4.6 0-0.1-0.1-0.1-0.1-0.2-0.8-1.4-1.7-2.8-2.7-4.1-0.1-0.1-0.2-0.3-0.3-0.4-0.5-0.6-0.9-1.1-1.4-1.7 0-0.1-0.1-0.1-0.1-0.2-0.5-0.6-1-1.1-1.6-1.6l-0.4-0.4c-0.5-0.5-1.1-1-1.6-1.5l-0.1-0.1c-0.6-0.5-1.2-1-1.9-1.4-0.1-0.1-0.3-0.2-0.4-0.3-1.4-1-2.8-1.8-4.3-2.6l-0.1-0.1c-1.6-0.8-3.2-1.5-4.9-2-1.6-0.5-3.3-1-5-1.2-0.1 0-0.2 0-0.3-0.1l-2.4-0.3h-0.3c-0.7-0.1-1.3-0.1-2-0.1H640.1c-19.9 0-36 16.1-36 36s16.1 36 36 36h165L487.6 487.6c-14.1 14.1-14.1 36.9 0 50.9 7 7 16.2 10.5 25.5 10.5 9.2 0 18.4-3.5 25.5-10.5L856 221v162.8c0 19.9 16.1 36 36 36s36-16.1 36-36V134.1c0-0.8 0-1.7-0.1-2.5z"
+        p-id="11369"></path>
+</svg>

projects/app/src/components/Icon/icons/common/viewLight.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701420318127" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="13238" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M987.52 367.36c-108.16-146.56-266.88-239.36-444.16-239.36-177.28 0-335.36 92.8-444.16 239.36-48 64.64-48 160.64 0 225.28C208 739.2 366.72 832 543.36 832c177.28 0 335.36-92.8 444.16-239.36C1035.52 528 1035.52 432 987.52 367.36zM939.52 539.52C839.04 684.8 694.4 768 542.72 768c-151.68 0-296.32-83.2-396.8-228.48C121.6 505.6 121.6 455.04 145.92 420.48 246.4 275.2 391.04 192 542.72 192c151.68 0 296.32 83.2 396.8 228.48C963.2 455.04 963.2 505.6 939.52 539.52zM544 256C420.48 256 320 356.48 320 480S420.48 704 544 704 768 603.52 768 480 667.52 256 544 256zM544 640C455.68 640 384 568.32 384 480S455.68 320 544 320C632.32 320 704 391.68 704 480S632.32 640 544 640z" p-id="13239"></path></svg>

projects/app/src/components/Icon/icons/core/app/variable/input.svg

@@ -0,0 +1,17 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701144443685"
+    class="icon" viewBox="0 0 1170 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="15096"
+    xmlns:xlink="http://www.w3.org/1999/xlink" width="146.25" height="128">
+    <path
+        d="M1024 128a39.862857 39.862857 0 0 1 36.571429 42.788571v682.422858a39.862857 39.862857 0 0 1-36.571429 42.788571H146.285714a39.862857 39.862857 0 0 1-36.571428-42.788571V170.788571A39.862857 39.862857 0 0 1 146.285714 128h877.714286M1024 0H146.285714C65.462857 0 0 76.434286 0 170.788571v682.422858C0 947.565714 65.462857 1024 146.285714 1024h877.714286c80.822857 0 146.285714-76.434286 146.285714-170.788571V170.788571C1170.285714 76.434286 1104.822857 0 1024 0z"
+        p-id="15097"></path>
+    <path
+        d="M310.857143 292.571429A54.857143 54.857143 0 0 0 256 347.428571v329.142858a54.857143 54.857143 0 0 0 109.714286 0v-329.142858A54.857143 54.857143 0 0 0 310.857143 292.571429z"
+        p-id="15098"></path>
+    <path
+        d="M365.714286 292.571429H256a54.857143 54.857143 0 0 0 0 109.714285h109.714286a54.857143 54.857143 0 0 0 0-109.714285zM365.714286 621.714286H256a54.857143 54.857143 0 0 0 0 109.714285h109.714286a54.857143 54.857143 0 0 0 0-109.714285z"
+        p-id="15099"></path>
+    <path
+        d="M556.251429 621.714286a54.857143 54.857143 0 1 0 54.857142 54.857143 54.857143 54.857143 0 0 0-54.857142-54.857143zM746.788571 621.714286a54.857143 54.857143 0 1 0 54.857143 54.857143 54.857143 54.857143 0 0 0-54.857143-54.857143z"
+        p-id="15100"></path>
+</svg>

projects/app/src/components/Icon/icons/core/app/variable/select.svg

@@ -0,0 +1,8 @@
+<svg viewBox="0 0 17 16" xmlns="http://www.w3.org/2000/svg">
+    <path fill-rule="evenodd" clip-rule="evenodd"
+        d="M8.89233 4.66667H3.77428C3.42285 4.66666 3.11966 4.66665 2.86995 4.68705C2.60639 4.70859 2.34424 4.75613 2.09199 4.88466C1.71567 5.07641 1.4097 5.38237 1.21796 5.75869C1.08943 6.01095 1.04188 6.27309 1.02035 6.53665C0.999949 6.78636 0.999959 7.08953 0.99997 7.44095V12.559C0.999959 12.9105 0.999949 13.2137 1.02035 13.4634C1.04188 13.7269 1.08943 13.9891 1.21796 14.2413C1.4097 14.6176 1.71567 14.9236 2.09199 15.1154C2.34424 15.2439 2.60639 15.2914 2.86995 15.313C3.11965 15.3334 3.4228 15.3334 3.77421 15.3333H8.89232C9.24372 15.3334 9.54696 15.3334 9.79666 15.313C10.0602 15.2914 10.3224 15.2439 10.5746 15.1154C10.9509 14.9236 11.2569 14.6176 11.4487 14.2413C11.5772 13.9891 11.6247 13.7269 11.6463 13.4634C11.6667 13.2136 11.6667 12.9105 11.6666 12.559V7.44099C11.6667 7.08955 11.6667 6.78637 11.6463 6.53665C11.6247 6.27309 11.5772 6.01095 11.4487 5.75869C11.2569 5.38237 10.9509 5.07641 10.5746 4.88466C10.3224 4.75613 10.0602 4.70859 9.79666 4.68705C9.54695 4.66665 9.24376 4.66666 8.89233 4.66667ZM9.13804 8.80474C9.39839 8.54439 9.39839 8.12228 9.13804 7.86193C8.87769 7.60159 8.45558 7.60159 8.19523 7.86193L5.66664 10.3905L4.80471 9.5286C4.54436 9.26825 4.12225 9.26825 3.8619 9.5286C3.60155 9.78895 3.60155 10.2111 3.8619 10.4714L5.19523 11.8047C5.45558 12.0651 5.87769 12.0651 6.13804 11.8047L9.13804 8.80474Z">
+    </path>
+    <path
+        d="M12.8923 0.666672H7.77427C7.42285 0.666661 7.11966 0.666651 6.86995 0.687053C6.60639 0.708587 6.34424 0.756131 6.09199 0.884661C5.71567 1.07641 5.40971 1.38237 5.21796 1.75869C5.08943 2.01095 5.04188 2.27309 5.02035 2.53665C5.00206 2.76051 5.00018 3.02733 4.99999 3.33336L8.92055 3.33335C9.2463 3.33327 9.59951 3.33319 9.90523 3.35816C10.2512 3.38644 10.7084 3.4564 11.1799 3.69667C11.8071 4.01625 12.3171 4.52618 12.6367 5.15339C12.8769 5.62493 12.9469 6.08208 12.9752 6.42809C13.0001 6.73382 13.0001 7.08702 13 7.41279L13 11.3333C13.306 11.3331 13.5728 11.3313 13.7967 11.313C14.0602 11.2914 14.3224 11.2439 14.5746 11.1154C14.9509 10.9236 15.2569 10.6176 15.4487 10.2413C15.5772 9.98907 15.6247 9.72692 15.6463 9.46336C15.6667 9.21366 15.6666 8.91051 15.6666 8.5591V3.44099C15.6666 3.08959 15.6667 2.78635 15.6463 2.53665C15.6247 2.27309 15.5772 2.01095 15.4487 1.75869C15.2569 1.38237 14.9509 1.07641 14.5746 0.884661C14.3224 0.756131 14.0602 0.708587 13.7967 0.687053C13.5469 0.666651 13.2438 0.666661 12.8923 0.666672Z">
+    </path>
+</svg>

projects/app/src/components/Icon/icons/core/app/variable/textarea.svg

@@ -0,0 +1,8 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701144482765"
+    class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="16956"
+    xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128">
+    <path
+        d="M870.4 709.7856c28.2624 0 51.2 24.4736 51.2 54.7328 0 28.0064-19.7632 51.2-45.2096 54.272L870.4 819.2H153.6c-28.2624 0-51.2-24.4736-51.2-54.6816 0-28.0576 19.7632-51.2 45.2096-54.3744L153.6 709.7856h716.8z m-159.2832-252.4672c28.2624 0 51.2 24.4736 51.2 54.6816 0 28.0576-19.7632 51.2-45.2096 54.3232l-5.9904 0.3584H153.6c-28.2624 0-51.2-24.4736-51.2-54.6816 0-28.0576 19.7632-51.2 45.2096-54.3232L153.6 457.3184h557.5168zM870.4 204.8c28.2624 0 51.2 24.4736 51.2 54.6816 0 28.0576-19.7632 51.2-45.2096 54.3744L870.4 314.2144H153.6c-28.2624 0-51.2-24.4736-51.2-54.7328 0-28.0064 19.7632-51.2 45.2096-54.272L153.6 204.8h716.8z"
+        p-id="16957"></path>
+</svg>

projects/app/src/components/Icon/icons/core/dataset/commonDataset.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1697622173220" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="2398" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M722.36246914 1024.8960632h-415.54678519c-120.26260543 0-218.12969876-97.86709333-218.12969877-218.12969875v-586.42583704c0-120.26260543 97.86709333-218.12969876 218.12969877-218.12969877h415.54678519c120.26260543 0 218.12969876 97.86709333 218.12969876 218.12969877v586.42583704c0 120.26260543-97.86709333 218.12969876-218.12969876 218.12969875z m-415.54678519-898.28010667c-51.65207703 0-93.72457086 42.07249383-93.72457086 93.72457088v586.42583704c0 51.65207703 42.07249383 93.72457086 93.72457086 93.72457086h415.54678519c51.65207703 0 93.72457086-42.07249383 93.72457086-93.72457086v-586.42583704c0-51.65207703-42.07249383-93.72457086-93.72457086-93.72457088h-415.54678519z" fill="#BDD2EF" p-id="2399"></path><path d="M684.0441363 599.76969482H338.27296395c-42.07249383 0-76.11885037-34.04635653-76.11885037-76.11885037s34.04635653-76.11885037 76.11885037-76.11885037h345.77117235c42.07249383 0 76.11885037 34.04635653 76.11885037 76.11885037s-34.17581037 76.11885037-76.11885037 76.11885037zM465.00826075 360.66847605h-126.7352968c-42.07249383 0-76.11885037-34.04635653-76.11885037-76.11885036s34.04635653-76.11885037 76.11885037-76.11885038h126.7352968c42.07249383 0 76.11885037 34.04635653 76.11885036 76.11885038s-34.04635653 76.11885037-76.11885036 76.11885036zM684.0441363 360.66847605h-25.37295013c-42.07249383 0-76.11885037-34.04635653-76.11885036-76.11885036s34.04635653-76.11885037 76.11885036-76.11885038h25.37295013c42.07249383 0 76.11885037 34.04635653 76.11885037 76.11885038s-34.17581037 76.11885037-76.11885037 76.11885036zM684.0441363 838.87091358H338.27296395c-42.07249383 0-76.11885037-34.04635653-76.11885037-76.11885038s34.04635653-76.11885037 76.11885037-76.11885037h345.77117235c42.07249383 0 76.11885037 34.04635653 76.11885037 76.11885037s-34.17581037 76.11885037-76.11885037 76.11885038z" fill="#2867CE" p-id="2400"></path></svg>

projects/app/src/components/Icon/icons/core/dataset/folderDataset.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1694141197423" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4891" xmlns:xlink="http://www.w3.org/1999/xlink" width="64" height="64"><path d="M855.04 385.024q19.456 2.048 38.912 10.24t33.792 23.04 21.504 37.376 2.048 54.272q-2.048 8.192-8.192 40.448t-14.336 74.24-18.432 86.528-19.456 76.288q-5.12 18.432-14.848 37.888t-25.088 35.328-36.864 26.112-51.2 10.24l-567.296 0q-21.504 0-44.544-9.216t-42.496-26.112-31.744-40.96-12.288-53.76l0-439.296q0-62.464 33.792-97.792t95.232-35.328l503.808 0q22.528 0 46.592 8.704t43.52 24.064 31.744 35.84 12.288 44.032l0 11.264-53.248 0q-40.96 0-95.744-0.512t-116.736-0.512-115.712-0.512-92.672-0.512l-47.104 0q-26.624 0-41.472 16.896t-23.04 44.544q-8.192 29.696-18.432 62.976t-18.432 61.952q-10.24 33.792-20.48 65.536-2.048 8.192-2.048 13.312 0 17.408 11.776 29.184t29.184 11.776q31.744 0 43.008-39.936l54.272-198.656q133.12 1.024 243.712 1.024l286.72 0z" fill="#FFCC66" p-id="4892"></path></svg>

projects/app/src/components/Icon/icons/core/dataset/modeEmbFTRerank.svg

@@ -0,0 +1,12 @@
+<svg t="1701068093720" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg"
+    p-id="14465" width="128" height="128">
+    <path
+        d="M413.696 719.36l-98.304 56.832-215.04-124.416v205.312l215.04 124.416L496.64 876.544l-82.944-47.616v-109.568z"
+        fill="#2B85FB" p-id="14466"></path>
+    <path
+        d="M751.616 435.2l-43.008-25.088-179.2 103.424-80.384-45.056 81.92-47.104V311.808l-215.04-124.416-215.04 124.416v248.32l41.984 24.064 173.568 100.352L414.72 627.712v-87.552l79.36 44.032V783.36l215.04 124.416 215.04-124.416v-248.32L751.616 435.2z"
+        fill="#2B85FB" p-id="14467"></path>
+    <path
+        d="M610.304 375.808l98.304-56.832 77.312 44.544V166.912l-215.04-124.416-174.08 100.352 212.992 122.88 0.512 110.08z"
+        fill="#2B85FB" p-id="14468"></path>
+</svg>

projects/app/src/components/Icon/icons/core/dataset/modeEmbedding.svg

@@ -0,0 +1,6 @@
+<svg t="1701067963240" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="6042"
+    width="128" height="128">
+    <path
+        d="M479.971202 0a95.99424 95.99424 0 0 1 92.538447 121.592704l304.365738 213.107214a95.99424 95.99424 0 1 1 35.83785 176.053437l-231.026138 355.690658a95.99424 95.99424 0 1 1-169.269844 70.715757L511.969282 927.944323c0-1.535908 0-3.071816 0.127992-4.543727l-276.143431-128.760274a95.99424 95.99424 0 1 1-129.528229-138.359699l-28.606283-274.031558A95.99424 95.99424 0 1 1 159.862408 216.307022l224.17855-119.224847L383.976961 95.99424A95.99424 95.99424 0 0 1 470.755755 0.447973L479.971202 0zM375.017499 600.731956l-126.264424 98.682079c3.519789 8.447493 5.759654 17.470952 6.783593 26.942384l0.319981 6.975581 280.431174 130.744155a96.634202 96.634202 0 0 1 11.32732-10.815351L469.987801 639.641622a159.15845 159.15845 0 0 1-84.986901-30.910146l-9.983401-7.99952z m467.427954-141.047537l-202.867828 32.254065a160.118393 160.118393 0 0 1-104.953702 138.423694L608.091515 831.950083c6.33562 0 12.543247 0.639962 18.558886 1.791892l229.746215-353.834769a96.378217 96.378217 0 0 1-9.791412-13.055217l-4.095755-7.16757zM163.190209 356.650601c-6.527608 6.33562-13.951163 11.775293-22.01468 16.063036l28.158311 267.695938c13.119213 1.279923 25.470472 5.119693 36.541807 11.199328l130.232187-101.753894a159.350439 159.350439 0 0 1-8.063517-119.864808L163.126212 356.650601zM535.903846 174.069556c-8.127512 5.759654-17.086975 10.303382-26.750395 13.439193l4.735716 136.055837a160.246385 160.246385 0 0 1 117.560946 104.889707l202.547847-32.126073c0.767954-3.83977 1.791892-7.679539 3.071816-11.32732L535.903846 174.069556zM406.375617 157.750535l-215.66706 114.681119a96.634202 96.634202 0 0 1 0.575966 26.750395l168.565886 75.067496c23.038618-26.110433 54.396736-44.66932 89.914605-51.452913l-4.799712-137.399756a96.058237 96.058237 0 0 1-32.062076-20.67076l-6.527609-7.039577z"
+        fill="#0A70B0" p-id="6043"></path>
+</svg>

projects/app/src/components/Icon/icons/core/dataset/modeEmbeddingRerank.svg

@@ -0,0 +1,11 @@
+<svg t="1701068031255" class="icon" viewBox="0 0 1407 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="8879"
+    width="128" height="128">
+    <path
+        d="M1202.775043 15.825987a48.650258 48.650258 0 0 0-68.579279 0 48.06411 48.06411 0 0 0 0 67.993131 607.835146 607.835146 0 0 1 0 855.189468 48.650258 48.650258 0 0 0 0 68.579279 53.339439 53.339439 0 0 0 34.582713 14.067544 48.06411 48.06411 0 0 0 33.996566-14.067544 703.377218 703.377218 0 0 0 0-991.761878zM96.714367 511.706926a603.145965 603.145965 0 0 1 175.844305-428.473955A48.298569 48.298569 0 0 0 205.151689 14.067544 703.377218 703.377218 0 0 0 58.614768 785.437894a689.895821 689.895821 0 0 0 152.398397 224.494562 47.477962 47.477962 0 0 0 33.996566 13.481396 48.650258 48.650258 0 0 0 33.996566-82.646823 599.042931 599.042931 0 0 1-182.29193-429.060103z"
+        fill="#5D78FD" p-id="8880"></path>
+    <path
+        d="M999.967945 218.633085a48.650258 48.650258 0 0 0-69.165426 67.993131 318.864339 318.864339 0 0 1 0 448.989125 48.650258 48.650258 0 0 0 0 68.579278 48.06411 48.06411 0 0 0 33.996565 14.067545 46.305667 46.305667 0 0 0 31.651975-13.481397 410.303377 410.303377 0 0 0 120.746422-293.073841 416.164854 416.164854 0 0 0-117.229536-293.073841z m-616.041213 293.073841a319.450487 319.450487 0 0 1 93.197481-226.839153A48.06411 48.06411 0 1 0 410.303377 218.633085a416.164854 416.164854 0 0 0 0 586.147682 47.477962 47.477962 0 0 0 33.410418 14.067544 48.650258 48.650258 0 0 0 35.168861-14.067544 48.06411 48.06411 0 0 0 0-68.579279 315.347453 315.347453 0 0 1-94.955924-224.494562z"
+        fill="#5D78FD" p-id="8881"></path>
+    <path d="M599.629078 511.706926a103.74814 103.74814 0 1 0 206.910132 0 103.74814 103.74814 0 0 0-206.910132 0z"
+        fill="#FEC743" p-id="8882"></path>
+</svg>

projects/app/src/components/Icon/icons/core/dataset/websiteDataset.svg

@@ -0,0 +1,14 @@
+<?xml version="1.0" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1701324062325"
+    class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="4651"
+    xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128">
+    <path d="M512 512m-512 0a512 512 0 1 0 1024 0 512 512 0 1 0-1024 0Z" fill="#c6dafe" p-id="4652"></path>
+    <path d="M486.4 665.6h51.2v153.6h-51.2z" fill="#515151" p-id="4653"
+        data-spm-anchor-id="a313x.search_index.0.i3.5ac83a811R1Mg6" class="selected"></path>
+    <path
+        d="M435.2 768h153.6v51.2h-153.6zM230.4 281.6v384h563.2V281.6H230.4z m0-51.2h563.2a51.2 51.2 0 0 1 51.2 51.2v384a51.2 51.2 0 0 1-51.2 51.2H230.4a51.2 51.2 0 0 1-51.2-51.2V281.6a51.2 51.2 0 0 1 51.2-51.2z"
+        fill="#515151" p-id="4654" data-spm-anchor-id="a313x.search_index.0.i1.5ac83a811R1Mg6" class="selected"></path>
+    <path
+        d="M662.272 388.864c6.144-21.5296 8.448-63.0272-29.952-68.3776-30.72-3.84-67.584 16.128-86.8352 29.2096-6.144-0.768-13.056-1.536-19.968-1.536-49.9712-0.7936-82.2016 16.128-109.8496 52.224-9.984 13.056-19.2 35.3536-22.2976 60.672 13.824-23.808 56.8576-67.584 102.1952-85.248 0 0-68.352 49.152-102.1952 118.3232v0.768c-1.5616 3.072-3.072 6.144-4.6336 9.984-33.024 82.2016-6.144 117.5552 19.2256 123.6992 23.04 6.144 55.3216-5.376 81.408-33.024 43.776 9.984 87.6288-1.536 103.7312-9.984 30.72-16.896 52.2496-46.848 56.8832-77.6192h-84.5312s-3.84 26.88-49.1776 26.88c-41.472 0-43.776-48.384-43.776-48.384h178.9952s3.072-52.2496-22.272-86.8352c-14.6176-19.2-33.8176-36.1216-60.6976-45.312 8.448-6.144 22.272-15.4112 34.56-18.4832 23.04-6.144 38.4256-2.304 48.4096 14.592 13.056 23.04-7.68 76.0832-7.68 76.0832s12.288-15.36 18.432-37.632z m-185.1648 203.6224c-35.328 29.2096-64.512 25.3696-76.0576 8.4736-9.984-15.36-11.52-42.2656 0-79.8976 5.376 14.592 13.824 28.416 26.112 40.704 15.36 15.36 32.256 25.344 49.9456 30.72z m-1.536-149.8368s1.536-34.56 39.168-37.632c32.3072-3.072 49.2032 11.52 54.5536 39.1936l-93.7472-1.536z"
+        fill="#2394FB" p-id="4655"></path>
+</svg>

projects/app/src/components/Icon/index.tsx

@@ -108,7 +108,20 @@ const iconPaths = {
   'core/chat/stopSpeechFill': () => import('./icons/core/chat/stopSpeechFill.svg'),
   'core/chat/stopSpeech': () => import('./icons/core/chat/stopSpeech.svg'),
   'core/chat/speaking': () => import('./icons/core/chat/speaking.svg'),
-  'core/chat/fileSelect': () => import('./icons/core/chat/fileSelect.svg')
+  'core/chat/fileSelect': () => import('./icons/core/chat/fileSelect.svg'),
+  'core/dataset/modeEmbedding': () => import('./icons/core/dataset/modeEmbedding.svg'),
+  'core/dataset/modeEmbeddingRerank': () => import('./icons/core/dataset/modeEmbeddingRerank.svg'),
+  'core/dataset/modeEmbFTRerank': () => import('./icons/core/dataset/modeEmbFTRerank.svg'),
+  'core/app/variable/input': () => import('./icons/core/app/variable/input.svg'),
+  'core/app/variable/textarea': () => import('./icons/core/app/variable/textarea.svg'),
+  'core/app/variable/select': () => import('./icons/core/app/variable/select.svg'),
+  'core/dataset/websiteDataset': () => import('./icons/core/dataset/websiteDataset.svg'),
+  'core/dataset/commonDataset': () => import('./icons/core/dataset/commonDataset.svg'),
+  'core/dataset/folderDataset': () => import('./icons/core/dataset/folderDataset.svg'),
+  'common/confirm/deleteTip': () => import('./icons/common/confirm/deleteTip.svg'),
+  'common/confirm/commonTip': () => import('./icons/common/confirm/commonTip.svg'),
+  'common/routePushLight': () => import('./icons/common/routePushLight.svg'),
+  'common/viewLight': () => import('./icons/common/viewLight.svg')
 };
 
 export type IconName = keyof typeof iconPaths;

projects/app/src/components/Layout/index.tsx

@@ -111,7 +111,7 @@ const Layout = ({ children }: { children: JSX.Element }) => {
           </>
         )}
       </Box>
-      <Loading loading={loading} zIndex={9999} />
+      <Loading loading={loading} zIndex={999999} />
       {!!userInfo && <UpdateInviteModal />}
     </>
   );

projects/app/src/components/Markdown/chat/Image.tsx

@@ -1,30 +1,33 @@
-import { Box, Flex } from '@chakra-ui/react';
+import { Box, Flex, Grid } from '@chakra-ui/react';
 import MdImage from '../img/Image';
 import { useMemo } from 'react';
 
 const ImageBlock = ({ images }: { images: string }) => {
   const formatData = useMemo(
     () =>
-      images.split('\n').map((item) => {
-        try {
-          return JSON.parse(item) as { src: string };
-        } catch (error) {
-          return { src: '' };
-        }
-      }),
+      images
+        .split('\n')
+        .filter((item) => item)
+        .map((item) => {
+          try {
+            return JSON.parse(item) as { src: string };
+          } catch (error) {
+            return { src: '' };
+          }
+        }),
     [images]
   );
 
   return (
-    <Flex alignItems={'center'} wrap={'wrap'} gap={4}>
+    <Grid gridTemplateColumns={['1fr', '1fr 1fr']} gap={4}>
       {formatData.map(({ src }) => {
         return (
-          <Box key={src} rounded={'md'} flex={'0 0 auto'} w={'120px'}>
+          <Box key={src} rounded={'md'} flex={'1 0 0'} minW={'120px'}>
             <MdImage src={src} />
           </Box>
         );
       })}
-    </Flex>
+    </Grid>
   );
 };
 

projects/app/src/components/Markdown/img/Image.tsx

@@ -29,7 +29,6 @@ const MdImage = ({ src }: { src?: string }) => {
         borderRadius={'md'}
         src={src}
         alt={''}
-        maxH={'150px'}
         fallbackSrc={'/imgs/errImg.png'}
         fallbackStrategy={'onError'}
         cursor={succeed ? 'pointer' : 'default'}
@@ -47,19 +46,17 @@ const MdImage = ({ src }: { src?: string }) => {
       />
       <Modal isOpen={isOpen} onClose={onClose} isCentered>
         <ModalOverlay />
-        <ModalContent maxW={'80vw'} maxH={'auto'}>
-          <Box>
-            <Image
-              borderRadius={'md'}
-              src={src}
-              alt={''}
-              w={'auto'}
-              h={'auto'}
-              fallbackSrc={'/imgs/errImg.png'}
-              fallbackStrategy={'onError'}
-              objectFit={'contain'}
-            />
-          </Box>
+        <ModalContent maxW={'auto'} w="auto" bg={'transparent'}>
+          <Image
+            borderRadius={'md'}
+            src={src}
+            alt={''}
+            w={'100%'}
+            maxH={'80vh'}
+            fallbackSrc={'/imgs/errImg.png'}
+            fallbackStrategy={'onError'}
+            objectFit={'contain'}
+          />
         </ModalContent>
         <ModalCloseButton bg={'myWhite.500'} zIndex={999999} />
       </Modal>

projects/app/src/components/MyModal/index.tsx

@@ -9,8 +9,9 @@ import {
   Box,
   Image
 } from '@chakra-ui/react';
+import MyIcon from '../Icon';
 
-interface Props extends ModalContentProps {
+export interface MyModalProps extends ModalContentProps {
   iconSrc?: string;
   title?: any;
   isCentered?: boolean;
@@ -28,7 +29,7 @@ const MyModal = ({
   w = 'auto',
   maxW = ['90vw', '600px'],
   ...props
-}: Props) => {
+}: MyModalProps) => {
   return (
     <Modal
       isOpen={isOpen}
@@ -42,7 +43,7 @@ const MyModal = ({
         minW={['90vw', '400px']}
         maxW={maxW}
         position={'relative'}
-        maxH={['80vh', '85vh']}
+        maxH={'85vh'}
         {...props}
       >
         {!title && onClose && <ModalCloseButton zIndex={1} />}
@@ -56,7 +57,15 @@ const MyModal = ({
             roundedTop={'lg'}
             py={'10px'}
           >
-            {iconSrc && <Image mr={3} objectFit={'contain'} alt="" src={iconSrc} w={'20px'} />}
+            {iconSrc && (
+              <>
+                {iconSrc.startsWith('/') ? (
+                  <Image mr={3} objectFit={'contain'} alt="" src={iconSrc} w={'20px'} />
+                ) : (
+                  <MyIcon mr={3} name={iconSrc as any} w={'20px'} />
+                )}
+              </>
+            )}
             {title}
             <Box flex={1} />
             {onClose && <ModalCloseButton position={'relative'} top={0} right={0} />}

projects/app/src/components/Slider/index.tsx

@@ -77,7 +77,7 @@ const MySlider = ({
         h={'18px'}
         borderRadius={'18px'}
         fontSize={'xs'}
-        transform={'translate(-50%, -170%)'}
+        transform={'translate(-50%, -155%)'}
         boxSizing={'border-box'}
       >
         {value}

projects/app/src/components/common/MyRadio/index.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
-import { Box, Flex, useTheme, Grid, type GridProps, theme } from '@chakra-ui/react';
-import type { StackProps } from '@chakra-ui/react';
+import { Box, Flex, useTheme, Grid, type GridProps, theme, Image } from '@chakra-ui/react';
 import MyIcon from '@/components/Icon';
+import { useTranslation } from 'next-i18next';
 
 // @ts-ignore
 interface Props extends GridProps {
@@ -20,6 +20,7 @@ const MyRadio = ({
   onChange,
   ...props
 }: Props) => {
+  const { t } = useTranslation();
   const theme = useTheme();
   return (
     <Grid gridGap={[3, 5]} fontSize={['sm', 'md']} {...props}>
@@ -35,15 +36,16 @@ const MyRadio = ({
           border={theme.borders.sm}
           borderWidth={'1.5px'}
           borderRadius={'md'}
-          bg={'myWhite.300'}
           position={'relative'}
           {...(value === item.value
             ? {
-                borderColor: 'myBlue.700'
+                borderColor: 'myBlue.500',
+                bg: 'myBlue.100'
               }
             : {
+                bg: 'myWhite.300',
                 _hover: {
-                  bg: 'white'
+                  borderColor: 'myBlue.500'
                 }
               })}
           _after={{
@@ -68,12 +70,20 @@ const MyRadio = ({
           }}
           onClick={() => onChange(item.value)}
         >
-          {!!item.icon && <MyIcon mr={'14px'} name={item.icon as any} w={iconSize} />}
+          {!!item.icon && (
+            <>
+              {item.icon.startsWith('/') ? (
+                <Image src={item.icon} mr={'14px'} w={iconSize} alt={''} />
+              ) : (
+                <MyIcon mr={'14px'} name={item.icon as any} w={iconSize} />
+              )}
+            </>
+          )}
           <Box pr={2}>
-            <Box>{item.title}</Box>
+            <Box>{t(item.title)}</Box>
             {!!item.desc && (
               <Box fontSize={'sm'} color={'myGray.500'}>
-                {item.desc}
+                {t(item.desc)}
               </Box>
             )}
           </Box>

projects/app/src/components/core/module/DatasetParamsModal.tsx

@@ -0,0 +1,152 @@
+import React, { useMemo, useState } from 'react';
+import { Box, Button, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
+import { useForm } from 'react-hook-form';
+import { QuestionOutlineIcon } from '@chakra-ui/icons';
+import MySlider from '@/components/Slider';
+import MyTooltip from '@/components/MyTooltip';
+import MyModal from '@/components/MyModal';
+import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
+import { useTranslation } from 'next-i18next';
+import { reRankModelList } from '@/web/common/system/staticData';
+
+import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants';
+import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constant';
+import MyRadio from '@/components/common/MyRadio';
+
+type DatasetParamsProps = {
+  similarity?: number;
+  limit?: number;
+  searchMode: `${DatasetSearchModeEnum}`;
+  searchEmptyText?: string;
+};
+
+const DatasetParamsModal = ({
+  searchEmptyText,
+  limit,
+  similarity,
+  searchMode = DatasetSearchModeEnum.embedding,
+  onClose,
+  onSuccess
+}: DatasetParamsProps & { onClose: () => void; onSuccess: (e: DatasetParamsProps) => void }) => {
+  const { t } = useTranslation();
+  const [refresh, setRefresh] = useState(false);
+  const { register, setValue, getValues, handleSubmit } = useForm<DatasetParamsProps>({
+    defaultValues: {
+      searchEmptyText,
+      limit,
+      similarity,
+      searchMode
+    }
+  });
+
+  const searchModeList = useMemo(() => {
+    const list = Object.values(DatasetSearchModeMap);
+    if (reRankModelList.length > 0) {
+      return list;
+    }
+    return list.slice(0, 1);
+  }, []);
+
+  return (
+    <MyModal
+      isOpen={true}
+      onClose={onClose}
+      iconSrc="/imgs/modal/params.svg"
+      title={'搜索参数调整'}
+      minW={['90vw', '500px']}
+      h={['90vh', 'auto']}
+      overflow={'unset'}
+      isCentered={searchEmptyText !== undefined}
+    >
+      <ModalBody flex={['1 0 0', 'auto']} overflow={'auto'}>
+        <MyRadio
+          gridGap={2}
+          gridTemplateColumns={'repeat(1,1fr)'}
+          list={searchModeList}
+          value={getValues('searchMode')}
+          onChange={(e) => {
+            setValue('searchMode', e as `${DatasetSearchModeEnum}`);
+            setRefresh(!refresh);
+          }}
+        />
+
+        {similarity !== undefined && (
+          <Box display={['block', 'flex']} py={8} mt={3}>
+            <Box flex={'0 0 100px'} mb={[8, 0]}>
+              {t('core.dataset.search.Min Similarity')}
+              <MyTooltip label={t('core.dataset.search.Min Similarity Tips')} forceShow>
+                <QuestionOutlineIcon ml={1} />
+              </MyTooltip>
+            </Box>
+            <MySlider
+              markList={[
+                { label: '0', value: 0 },
+                { label: '1', value: 1 }
+              ]}
+              min={0}
+              max={1}
+              step={0.01}
+              value={getValues(ModuleInputKeyEnum.datasetSimilarity) || 0.5}
+              onChange={(val) => {
+                setValue(ModuleInputKeyEnum.datasetSimilarity, val);
+                setRefresh(!refresh);
+              }}
+            />
+          </Box>
+        )}
+        {limit !== undefined && (
+          <Box display={['block', 'flex']} py={8}>
+            <Box flex={'0 0 100px'} mb={[8, 0]}>
+              {t('core.dataset.search.Top K')}
+            </Box>
+            <Box flex={1}>
+              <MySlider
+                markList={[
+                  { label: '1', value: 1 },
+                  { label: '30', value: 30 }
+                ]}
+                min={1}
+                max={30}
+                value={getValues(ModuleInputKeyEnum.datasetLimit) || 5}
+                onChange={(val) => {
+                  setValue(ModuleInputKeyEnum.datasetLimit, val);
+                  setRefresh(!refresh);
+                }}
+              />
+            </Box>
+          </Box>
+        )}
+        {searchEmptyText !== undefined && (
+          <Box display={['block', 'flex']} pt={3}>
+            <Box flex={'0 0 100px'} mb={[2, 0]}>
+              {t('core.dataset.search.Empty result response')}
+            </Box>
+            <Box flex={1}>
+              <Textarea
+                rows={5}
+                maxLength={500}
+                placeholder={t('core.dataset.search.Empty result response Tips')}
+                {...register('searchEmptyText')}
+              ></Textarea>
+            </Box>
+          </Box>
+        )}
+      </ModalBody>
+      <ModalFooter>
+        <Button variant={'base'} mr={3} onClick={onClose}>
+          {t('common.Close')}
+        </Button>
+        <Button
+          onClick={() => {
+            onClose();
+            handleSubmit(onSuccess)();
+          }}
+        >
+          {t('common.Done')}
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};
+
+export default DatasetParamsModal;

projects/app/src/components/core/module/DatasetSelectModal.tsx

@@ -7,32 +7,20 @@ import {
   ModalBody,
   ModalFooter,
   useTheme,
-  Textarea,
   Grid,
-  Divider,
-  Switch,
-  Image
+  Divider
 } from '@chakra-ui/react';
 import Avatar from '@/components/Avatar';
-import { useForm } from 'react-hook-form';
-import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import type { SelectedDatasetType } from '@fastgpt/global/core/module/api.d';
 import { useToast } from '@/web/common/hooks/useToast';
-import MySlider from '@/components/Slider';
 import MyTooltip from '@/components/MyTooltip';
-import MyModal from '@/components/MyModal';
 import MyIcon from '@/components/Icon';
 import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import { useTranslation } from 'next-i18next';
 import { useDatasetStore } from '@/web/core/dataset/store/dataset';
-import { feConfigs } from '@/web/common/system/staticData';
 import DatasetSelectContainer, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import EmptyTip from '@/components/EmptyTip';
-import { AppSimpleEditFormType } from '@fastgpt/global/core/app/type';
-import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants';
-
-type DatasetParamsProps = AppSimpleEditFormType['dataset'];
 
 export const DatasetSelectModal = ({
   isOpen,
@@ -135,9 +123,9 @@ export const DatasetSelectModal = ({
                   <MyTooltip
                     key={item._id}
                     label={
-                      item.type === DatasetTypeEnum.dataset
-                        ? t('dataset.Select Dataset')
-                        : t('dataset.Select Folder')
+                      item.type === DatasetTypeEnum.folder
+                        ? t('dataset.Select Folder')
+                        : t('dataset.Select Dataset')
                     }
                   >
                     <Card
@@ -152,7 +140,7 @@ export const DatasetSelectModal = ({
                       onClick={() => {
                         if (item.type === DatasetTypeEnum.folder) {
                           setParentId(item._id);
-                        } else if (item.type === DatasetTypeEnum.dataset) {
+                        } else {
                           const vectorModel = selectedDatasets[0]?.vectorModel?.model;
 
                           if (vectorModel && vectorModel !== item.vectorModel.model) {
@@ -222,127 +210,4 @@ export const DatasetSelectModal = ({
   );
 };
 
-export const DatasetParamsModal = ({
-  searchEmptyText,
-  limit,
-  similarity,
-  rerank,
-  onClose,
-  onChange
-}: DatasetParamsProps & { onClose: () => void; onChange: (e: DatasetParamsProps) => void }) => {
-  const [refresh, setRefresh] = useState(false);
-  const { register, setValue, getValues, handleSubmit } = useForm<DatasetParamsProps>({
-    defaultValues: {
-      searchEmptyText,
-      limit,
-      similarity,
-      rerank
-    }
-  });
-
-  return (
-    <MyModal
-      isOpen={true}
-      onClose={onClose}
-      iconSrc="/imgs/modal/params.svg"
-      title={'搜索参数调整'}
-      minW={['90vw', '600px']}
-    >
-      <Flex flexDirection={'column'}>
-        <ModalBody>
-          {feConfigs?.isPlus && (
-            <Box display={['block', 'flex']} py={5} pt={[0, 5]}>
-              <Box flex={'0 0 100px'} mb={[8, 0]}>
-                结果重排
-                <MyTooltip label={'将召回的结果进行进一步重排，可增加召回率'} forceShow>
-                  <QuestionOutlineIcon ml={1} />
-                </MyTooltip>
-              </Box>
-              <Switch
-                size={'lg'}
-                isChecked={getValues(ModuleInputKeyEnum.datasetStartReRank)}
-                onChange={(e) => {
-                  setValue(ModuleInputKeyEnum.datasetStartReRank, e.target.checked);
-                  setRefresh(!refresh);
-                }}
-              />
-            </Box>
-          )}
-          <Box display={['block', 'flex']} py={5} pt={[0, 5]}>
-            <Box flex={'0 0 100px'} mb={[8, 0]}>
-              相似度
-              <MyTooltip
-                label={'不同索引模型的相似度有区别，请通过搜索测试来选择合适的数值'}
-                forceShow
-              >
-                <QuestionOutlineIcon ml={1} />
-              </MyTooltip>
-            </Box>
-            <MySlider
-              markList={[
-                { label: '0', value: 0 },
-                { label: '1', value: 1 }
-              ]}
-              min={0}
-              max={1}
-              step={0.01}
-              value={getValues(ModuleInputKeyEnum.datasetSimilarity)}
-              onChange={(val) => {
-                setValue(ModuleInputKeyEnum.datasetSimilarity, val);
-                setRefresh(!refresh);
-              }}
-            />
-          </Box>
-          <Box display={['block', 'flex']} py={8}>
-            <Box flex={'0 0 100px'} mb={[8, 0]}>
-              单次搜索数量
-            </Box>
-            <Box flex={1}>
-              <MySlider
-                markList={[
-                  { label: '1', value: 1 },
-                  { label: '20', value: 20 }
-                ]}
-                min={1}
-                max={20}
-                value={getValues(ModuleInputKeyEnum.datasetLimit)}
-                onChange={(val) => {
-                  setValue(ModuleInputKeyEnum.datasetLimit, val);
-                  setRefresh(!refresh);
-                }}
-              />
-            </Box>
-          </Box>
-          <Box display={['block', 'flex']} pt={3}>
-            <Box flex={'0 0 100px'} mb={[2, 0]}>
-              空搜索回复
-            </Box>
-            <Box flex={1}>
-              <Textarea
-                rows={5}
-                maxLength={500}
-                placeholder={`若填写该内容，没有搜索到对应内容时，将直接回复填写的内容。\n为了连贯上下文，${feConfigs?.systemTitle} 会取部分上一个聊天的搜索记录作为补充，因此在连续对话时，该功能可能会失效。`}
-                {...register('searchEmptyText')}
-              ></Textarea>
-            </Box>
-          </Box>
-        </ModalBody>
-        <ModalFooter>
-          <Button variant={'base'} mr={3} onClick={onClose}>
-            取消
-          </Button>
-          <Button
-            onClick={() => {
-              onClose();
-              handleSubmit(onChange)();
-            }}
-          >
-            完成
-          </Button>
-        </ModalFooter>
-      </Flex>
-    </MyModal>
-  );
-};
-
 export default DatasetSelectModal;

projects/app/src/components/core/module/Flow/ModuleTemplateList.tsx

@@ -1,14 +1,5 @@
 import React, { useCallback, useMemo } from 'react';
-import {
-  Box,
-  Flex,
-  Accordion,
-  AccordionItem,
-  AccordionButton,
-  AccordionPanel,
-  AccordionIcon,
-  useTheme
-} from '@chakra-ui/react';
+import { Box, Flex } from '@chakra-ui/react';
 import type {
   FlowModuleTemplateType,
   moduleTemplateListType

projects/app/src/components/core/module/Flow/components/modules/VariableEdit.tsx

@@ -1,4 +1,4 @@
-import React, { useEffect, useState } from 'react';
+import React, { useEffect, useMemo, useState } from 'react';
 import {
   Box,
   Button,
@@ -54,13 +54,18 @@ const VariableEdit = ({
 
   const VariableTypeList = [
     {
-      label: t('core.module.variable.text type'),
-      icon: 'settingLight',
+      label: t('core.module.variable.input type'),
+      icon: 'core/app/variable/input',
       key: VariableInputEnum.input
     },
+    {
+      label: t('core.module.variable.textarea type'),
+      icon: 'core/app/variable/textarea',
+      key: VariableInputEnum.textarea
+    },
     {
       label: t('core.module.variable.select type'),
-      icon: 'settingLight',
+      icon: 'core/app/variable/select',
       key: VariableInputEnum.select
     }
   ];
@@ -94,6 +99,13 @@ const VariableEdit = ({
     }
   };
 
+  const formatVariables = useMemo(() => {
+    return variables.map((item) => ({
+      ...item,
+      icon: VariableTypeList.find((type) => type.key === item.type)?.icon
+    }));
+  }, [variables]);
+
   return (
     <Box>
       <Flex alignItems={'center'}>
@@ -114,12 +126,13 @@ const VariableEdit = ({
           +&ensp;{t('common.Add New')}
         </Flex>
       </Flex>
-      {variables.length > 0 && (
+      {formatVariables.length > 0 && (
         <Box mt={2} borderRadius={'lg'} overflow={'hidden'} borderWidth={'1px'} borderBottom="none">
           <TableContainer>
             <Table bg={'white'}>
               <Thead>
                 <Tr>
+                  <Th w={'18px !important'} p={0} />
                   <Th>{t('core.module.variable.variable name')}</Th>
                   <Th>{t('core.module.variable.key')}</Th>
                   <Th>{t('common.Require Input')}</Th>
@@ -127,9 +140,12 @@ const VariableEdit = ({
                 </Tr>
               </Thead>
               <Tbody>
-                {variables.map((item, index) => (
+                {formatVariables.map((item) => (
                   <Tr key={item.id}>
-                    <Td>{item.label} </Td>
+                    <Td textAlign={'center'} p={0} pl={3}>
+                      <MyIcon name={item.icon as any} w={'14px'} color={'myGray.500'} />
+                    </Td>
+                    <Td>{item.label}</Td>
                     <Td>{item.key}</Td>
                     <Td>{item.required ? '✔' : ''}</Td>
                     <Td>
@@ -190,20 +206,24 @@ const VariableEdit = ({
           <Box mt={5} mb={2}>
             {t('core.module.Field Type')}
           </Box>
-          <Grid gridTemplateColumns={'repeat(2,130px)'} gridGap={4}>
+          <Grid gridTemplateColumns={'repeat(3,1fr)'} gridGap={4}>
             {VariableTypeList.map((item) => (
               <Flex
                 key={item.key}
-                px={4}
-                py={1}
+                px={3}
+                py={3}
                 border={theme.borders.base}
                 borderRadius={'md'}
                 cursor={'pointer'}
                 {...(item.key === getValuesEdit('variable.type')
                   ? {
-                      bg: 'myWhite.600'
+                      bg: 'myBlue.100',
+                      borderColor: 'myBlue.600',
+                      color: 'myBlue.600',
+                      fontWeight: 'bold'
                     }
                   : {
+                      color: 'myGray.600',
                       _hover: {
                         boxShadow: 'md'
                       },
@@ -214,7 +234,7 @@ const VariableEdit = ({
                     })}
               >
                 <MyIcon name={item.icon as any} w={'16px'} />
-                <Box ml={3}>{item.label}</Box>
+                <Box ml={2}>{item.label}</Box>
               </Flex>
             ))}
           </Grid>
@@ -225,14 +245,14 @@ const VariableEdit = ({
                 {t('core.module.variable.text max length')}
               </Box>
               <Box>
-                <NumberInput max={100} min={1} step={1} position={'relative'}>
+                <NumberInput max={500} min={1} step={1} position={'relative'}>
                   <NumberInputField
                     {...registerEdit('variable.maxLen', {
                       min: 1,
-                      max: 100,
+                      max: 500,
                       valueAsNumber: true
                     })}
-                    max={100}
+                    max={500}
                   />
                   <NumberInputStepper>
                     <NumberIncrementStepper />
@@ -258,16 +278,18 @@ const VariableEdit = ({
                         })}
                       />
                     </FormControl>
-                    <MyIcon
-                      ml={3}
-                      name={'delete'}
-                      w={'16px'}
-                      cursor={'pointer'}
-                      p={2}
-                      borderRadius={'lg'}
-                      _hover={{ bg: 'red.100' }}
-                      onClick={() => removeEnums(i)}
-                    />
+                    {selectEnums.length > 1 && (
+                      <MyIcon
+                        ml={3}
+                        name={'delete'}
+                        w={'16px'}
+                        cursor={'pointer'}
+                        p={2}
+                        borderRadius={'lg'}
+                        _hover={{ bg: 'red.100' }}
+                        onClick={() => removeEnums(i)}
+                      />
+                    )}
                   </Flex>
                 ))}
               </Box>

projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx

@@ -1,4 +1,4 @@
-import React, { useMemo, useState } from 'react';
+import React, { useCallback, useEffect, useMemo, useState } from 'react';
 import type { SelectAppItemType } from '@fastgpt/global/core/module/type';
 import type { FlowNodeInputItemType } from '@fastgpt/global/core/module/node/type';
 import {
@@ -36,11 +36,13 @@ import type { SelectedDatasetType } from '@fastgpt/global/core/module/api.d';
 import { useQuery } from '@tanstack/react-query';
 import type { EditFieldModeType, EditFieldType } from '../modules/FieldEditModal';
 import { feConfigs } from '@/web/common/system/staticData';
+import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
 
 const FieldEditModal = dynamic(() => import('../modules/FieldEditModal'));
 const SelectAppModal = dynamic(() => import('../../SelectAppModal'));
 const AIChatSettingsModal = dynamic(() => import('../../../AIChatSettingsModal'));
 const DatasetSelectModal = dynamic(() => import('../../../DatasetSelectModal'));
+const DatasetParamsModal = dynamic(() => import('../../../DatasetParamsModal'));
 
 export const Label = React.memo(function Label({
   moduleId,
@@ -232,6 +234,9 @@ const RenderInput = ({
                 {item.type === FlowNodeInputTypeEnum.selectDataset && (
                   <SelectDatasetRender item={item} moduleId={moduleId} />
                 )}
+                {item.type === FlowNodeInputTypeEnum.selectDatasetParamsModal && (
+                  <SelectDatasetParamsRender item={item} inputs={sortInputs} moduleId={moduleId} />
+                )}
                 {item.type === FlowNodeInputTypeEnum.custom && CustomComponent[item.key] && (
                   <>{CustomComponent[item.key]({ ...item })}</>
                 )}
@@ -251,7 +256,7 @@ type RenderProps = {
   moduleId: string;
 };
 
-var NumberInputRender = React.memo(function NumberInputRender({ item, moduleId }: RenderProps) {
+const NumberInputRender = React.memo(function NumberInputRender({ item, moduleId }: RenderProps) {
   return (
     <NumberInput
       defaultValue={item.value}
@@ -278,7 +283,7 @@ var NumberInputRender = React.memo(function NumberInputRender({ item, moduleId }
   );
 });
 
-var TextInputRender = React.memo(function TextInputRender({ item, moduleId }: RenderProps) {
+const TextInputRender = React.memo(function TextInputRender({ item, moduleId }: RenderProps) {
   return (
     <Input
       placeholder={item.placeholder}
@@ -298,7 +303,7 @@ var TextInputRender = React.memo(function TextInputRender({ item, moduleId }: Re
   );
 });
 
-var SwitchRender = React.memo(function SwitchRender({ item, moduleId }: RenderProps) {
+const SwitchRender = React.memo(function SwitchRender({ item, moduleId }: RenderProps) {
   return (
     <Switch
       size={'lg'}
@@ -318,7 +323,7 @@ var SwitchRender = React.memo(function SwitchRender({ item, moduleId }: RenderPr
   );
 });
 
-var TextareaRender = React.memo(function TextareaRender({ item, moduleId }: RenderProps) {
+const TextareaRender = React.memo(function TextareaRender({ item, moduleId }: RenderProps) {
   return (
     <Textarea
       rows={5}
@@ -340,7 +345,7 @@ var TextareaRender = React.memo(function TextareaRender({ item, moduleId }: Rend
   );
 });
 
-var SelectRender = React.memo(function SelectRender({ item, moduleId }: RenderProps) {
+const SelectRender = React.memo(function SelectRender({ item, moduleId }: RenderProps) {
   return (
     <MySelect
       width={'100%'}
@@ -361,7 +366,7 @@ var SelectRender = React.memo(function SelectRender({ item, moduleId }: RenderPr
   );
 });
 
-var SliderRender = React.memo(function SliderRender({ item, moduleId }: RenderProps) {
+const SliderRender = React.memo(function SliderRender({ item, moduleId }: RenderProps) {
   return (
     <Box pt={5} pb={4} px={2}>
       <MySlider
@@ -387,7 +392,7 @@ var SliderRender = React.memo(function SliderRender({ item, moduleId }: RenderPr
   );
 });
 
-var AISetting = React.memo(function AISetting({ inputs = [], moduleId }: RenderProps) {
+const AISetting = React.memo(function AISetting({ inputs = [], moduleId }: RenderProps) {
   const { t } = useTranslation();
   const chatModulesData = useMemo(() => {
     const obj: Record<string, any> = {};
@@ -440,15 +445,15 @@ var AISetting = React.memo(function AISetting({ inputs = [], moduleId }: RenderP
   );
 });
 
-var SelectChatModelRender = React.memo(function SelectChatModelRender({
+const SelectChatModelRender = React.memo(function SelectChatModelRender({
   inputs = [],
   item,
   moduleId
 }: RenderProps) {
   const modelList = chatModelList || [];
 
-  function onChangeModel(e: string) {
-    {
+  const onChangeModel = useCallback(
+    (e: string) => {
       onChangeNode({
         moduleId,
         type: 'updateInput',
@@ -477,8 +482,9 @@ var SelectChatModelRender = React.memo(function SelectChatModelRender({
           value: model.maxResponse / 2
         }
       });
-    }
-  }
+    },
+    [inputs, item, modelList, moduleId]
+  );
 
   const list = modelList.map((item) => {
     const priceStr = `(${formatPrice(item.price, 1000)}元/1k Tokens)`;
@@ -489,9 +495,11 @@ var SelectChatModelRender = React.memo(function SelectChatModelRender({
     };
   });
 
-  if (!item.value && list.length > 0) {
-    onChangeModel(list[0].value);
-  }
+  useEffect(() => {
+    if (!item.value && list.length > 0) {
+      onChangeModel(list[0].value);
+    }
+  }, [item.value, list, onChangeModel]);
 
   return (
     <MySelect
@@ -504,7 +512,10 @@ var SelectChatModelRender = React.memo(function SelectChatModelRender({
   );
 });
 
-var SelectDatasetRender = React.memo(function SelectDatasetRender({ item, moduleId }: RenderProps) {
+const SelectDatasetRender = React.memo(function SelectDatasetRender({
+  item,
+  moduleId
+}: RenderProps) {
   const theme = useTheme();
   const { mode } = useFlowProviderStore();
   const { allDatasets, loadAllDatasets } = useDatasetStore();
@@ -572,7 +583,7 @@ var SelectDatasetRender = React.memo(function SelectDatasetRender({ item, module
   );
 });
 
-var SelectAppRender = React.memo(function SelectAppRender({ item, moduleId }: RenderProps) {
+const SelectAppRender = React.memo(function SelectAppRender({ item, moduleId }: RenderProps) {
   const { filterAppIds } = useFlowProviderStore();
   const theme = useTheme();
 
@@ -622,3 +633,63 @@ var SelectAppRender = React.memo(function SelectAppRender({ item, moduleId }: Re
     </>
   );
 });
+
+const SelectDatasetParamsRender = React.memo(function SelectDatasetParamsRender({
+  inputs = [],
+  moduleId
+}: RenderProps) {
+  const { t } = useTranslation();
+  const [data, setData] = useState({
+    searchMode: DatasetSearchModeEnum.embedding,
+    limit: 5,
+    similarity: 0.5
+  });
+
+  const { isOpen, onOpen, onClose } = useDisclosure();
+
+  useEffect(() => {
+    inputs.forEach((input) => {
+      // @ts-ignore
+      if (data[input.key] !== undefined) {
+        setData((state) => ({
+          ...state,
+          [input.key]: input.value
+        }));
+      }
+    });
+  }, [inputs]);
+
+  return (
+    <>
+      <Button
+        variant={'base'}
+        leftIcon={<MyIcon name={'settingLight'} w={'14px'} />}
+        onClick={onOpen}
+      >
+        {t('core.dataset.search.Params Setting')}
+      </Button>
+      {isOpen && (
+        <DatasetParamsModal
+          {...data}
+          onClose={onClose}
+          onSuccess={(e) => {
+            for (let key in e) {
+              const item = inputs.find((input) => input.key === key);
+              if (!item) continue;
+              onChangeNode({
+                moduleId,
+                type: 'updateInput',
+                key,
+                value: {
+                  ...item,
+                  //@ts-ignore
+                  value: e[key]
+                }
+              });
+            }
+          }}
+        />
+      )}
+    </>
+  );
+});

projects/app/src/components/support/apikey/Table.tsx

@@ -82,14 +82,16 @@ const ApiKeyTable = ({ tips, appId }: { tips: string; appId?: string }) => {
             <Box fontSize={['md', 'xl']} fontWeight={'bold'}>
               API 秘钥管理
             </Box>
-            <Link
-              href={feConfigs.openAPIDocUrl || 'https://doc.fastgpt.run/docs/development/openapi'}
-              target={'_blank'}
-              ml={1}
-              color={'myBlue.600'}
-            >
-              查看文档
-            </Link>
+            {feConfigs.docUrl && (
+              <Link
+                href={feConfigs.openAPIDocUrl || `${feConfigs.docUrl}/docs/development/openapi`}
+                target={'_blank'}
+                ml={1}
+                color={'myBlue.600'}
+              >
+                查看文档
+              </Link>
+            )}
           </Flex>
           <Box fontSize={'sm'} color={'myGray.600'}>
             {tips}

projects/app/src/components/support/permission/Radio/index.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import MyRadio from '@/components/Radio';
+import MyRadio from '@/components/common/MyRadio';
 import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
 import { useTranslation } from 'next-i18next';
 

projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx

@@ -51,8 +51,8 @@ function EditModal({
       try {
         const src = await compressImgFileAndUpload({
           file,
-          maxW: 100,
-          maxH: 100
+          maxW: 300,
+          maxH: 300
         });
         setValue('avatar', src);
         setRefresh((state) => !state);

projects/app/src/constants/dataset.ts

@@ -14,7 +14,8 @@ export const defaultDatasetDetail: DatasetItemType = {
   type: 'dataset',
   avatar: '/icon/logo.svg',
   name: '',
-  tags: [],
+  intro: '',
+  status: 'active',
   permission: 'private',
   isOwner: false,
   canWrite: false,
@@ -24,7 +25,6 @@ export const defaultDatasetDetail: DatasetItemType = {
 
 export const defaultCollectionDetail: DatasetCollectionItemType = {
   _id: '',
-  userId: '',
   teamId: '',
   tmbId: '',
   datasetId: {
@@ -37,7 +37,8 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
     type: 'dataset',
     avatar: '/icon/logo.svg',
     name: '',
-    tags: [],
+    intro: '',
+    status: 'active',
     permission: 'private',
     vectorModel: defaultVectorModels[0].model,
     agentModel: defaultQAModels[0].model
@@ -46,8 +47,10 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
   name: '',
   type: 'file',
   updateTime: new Date(),
-  metadata: {},
   canWrite: false,
   sourceName: '',
-  sourceId: ''
+  sourceId: '',
+  createTime: new Date(),
+  trainingType: 'chunk',
+  chunkSize: 0
 };

projects/app/src/global/common/api/systemRes.d.ts

@@ -4,7 +4,8 @@ import type {
   LLMModelItemType,
   VectorModelItemType,
   AudioSpeechModels,
-  WhisperModelType
+  WhisperModelType,
+  ReRankModelItemType
 } from '@fastgpt/global/core/ai/model.d';
 
 import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d';
@@ -19,6 +20,7 @@ export type ConfigFileType = {
   ExtractModels: FunctionModelItemType[];
   QGModels: LLMModelItemType[];
   VectorModels: VectorModelItemType[];
+  ReRankModels: ReRankModelItemType[];
   AudioSpeechModels: AudioSpeechModelType[];
   WhisperModel: WhisperModelType;
 };
@@ -29,6 +31,7 @@ export type InitDateResponse = {
   extractModels: FunctionModelItemType[];
   vectorModels: VectorModelItemType[];
   audioSpeechModels: AudioSpeechModels[];
+  reRankModels: ReRankModelItemType[];
   feConfigs: FeConfigsType;
   priceMd: string;
   systemVersion: string;

projects/app/src/global/core/api/datasetReq.d.ts

@@ -1,4 +1,8 @@
-import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetCollectionTrainingModeEnum,
+  DatasetCollectionTypeEnum,
+  DatasetTypeEnum
+} from '@fastgpt/global/core/dataset/constant';
 import type { RequestPaging } from '@/types';
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
 import type { SearchTestItemType } from '@/types/core/dataset';
@@ -8,15 +12,6 @@ import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'
 import type { LLMModelItemType } from '@fastgpt/global/core/ai/model.d';
 
 /* ===== dataset ===== */
-export type DatasetUpdateParams = {
-  id: string;
-  parentId?: string;
-  tags?: string[];
-  name?: string;
-  avatar?: string;
-  permission?: `${PermissionTypeEnum}`;
-  agentModel?: LLMModelItemType;
-};
 
 /* ======= collections =========== */
 export type GetDatasetCollectionsProps = RequestPaging & {
@@ -26,19 +21,11 @@ export type GetDatasetCollectionsProps = RequestPaging & {
   simple?: boolean;
   selectFolder?: boolean;
 };
-export type CreateDatasetCollectionParams = {
-  datasetId: string;
-  parentId?: string;
-  name: string;
-  type: `${DatasetCollectionTypeEnum}`;
-  metadata?: DatasetCollectionSchemaType['metadata'];
-  updateTime?: string;
-};
+
 export type UpdateDatasetCollectionParams = {
   id: string;
   parentId?: string;
   name?: string;
-  metadata?: DatasetCollectionSchemaType['metadata'];
 };
 
 /* ==== data ===== */

projects/app/src/global/core/app/constants.ts

@@ -1,4 +1,5 @@
 import { AppSimpleEditConfigTemplateType } from '@fastgpt/global/core/app/type.d';
+import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
 
 export const SimpleModeTemplate_FastGPT_Universal: AppSimpleEditConfigTemplateType = {
   id: 'fastgpt-universal',
@@ -17,7 +18,7 @@ export const SimpleModeTemplate_FastGPT_Universal: AppSimpleEditConfigTemplateTy
       datasets: true,
       similarity: true,
       limit: true,
-      rerank: true,
+      searchMode: DatasetSearchModeEnum.embedding,
       searchEmptyText: true
     },
     userGuide: {

projects/app/src/global/core/dataset/api.d.ts

@@ -1,5 +1,9 @@
 import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api';
-import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetSearchModeEnum,
+  DatasetTypeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constant';
 import {
   DatasetDataIndexItemType,
   SearchDataResponseItemType
@@ -8,8 +12,9 @@ import {
 /* ================= dataset ===================== */
 export type CreateDatasetParams = {
   parentId?: string;
+  type: `${DatasetTypeEnum}`;
   name: string;
-  tags: string;
+  intro: string;
   avatar: string;
   vectorModel?: string;
   agentModel?: string;
@@ -43,7 +48,7 @@ export type SearchTestProps = {
   datasetId: string;
   text: string;
   limit?: number;
-  rerank?: boolean;
+  searchMode?: `${DatasetSearchModeEnum}`;
 };
 export type SearchTestResponse = {
   list: SearchDataResponseItemType[];

projects/app/src/global/core/dataset/type.d.ts

@@ -16,7 +16,8 @@ export type DatasetCollectionsListItemType = {
   updateTime: Date;
   dataAmount: number;
   trainingAmount: number;
-  metadata: DatasetCollectionSchemaType['metadata'];
+  fileId?: string;
+  rawLink?: string;
   canWrite: boolean;
 };
 

projects/app/src/global/core/prompt/AIChat.ts

@@ -27,57 +27,56 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [
   {
     title: '标准模板',
     desc: '',
-    value: `你的背景知识:
+    value: `你的知识库:
 """
 {{quote}}
 """
-对话要求：
-1. 背景知识是最新的实时的信息，使用背景知识回答问题。
-2. 优先使用背景知识的内容回答我的问题，答案应与背景知识严格一致。
-3. 背景知识无法回答我的问题时，可以忽略背景知识，根据你的知识来自由回答。
-4. 使用对话的风格，自然的回答问题。包含markdown内容，需按markdown格式返回。
+回答要求：
+1. 优先使用知识库内容回答问题。
+2. 你可以回答我不知道。
+3. 不要提及你是从知识库获取的知识。
+4. 知识库包含 markdown 内容时，按 markdown 格式返回。
 我的问题是:"{{question}}"`
   },
   {
     title: '问答模板',
     desc: '',
-    value: `你的背景知识:
+    value: `你的知识库:
 """
 {{quote}}
 """
-对话要求：
-1. 背景知识是最新的实时的信息，使用背景知识回答问题，其中 instruction 是相关介绍，output 是预期回答或补充。
-2. 优先使用背景知识的内容回答我的问题，答案应与背景知识严格一致。
-3. 背景知识无法回答我的问题时，可以忽略背景知识，根据你的知识来自由回答。
-4. 使用对话的风格，自然的回答问题。包含markdown内容，需按markdown格式返回。
+回答要求：
+1. 优先使用知识库内容回答问题，其中 instruction 是相关介绍，output 是预期回答或补充。
+2. 你可以回答我不知道。
+3. 不要提及你是从知识库获取的知识。
+4. 知识库包含 markdown 内容时，按 markdown 格式返回。
 我的问题是:"{{question}}"`
   },
   {
     title: '标准严格模板',
     desc: '',
-    value: `你的背景知识:
+    value: `你的知识库:
 """
 {{quote}}
 """
-对话要求：
-1. 背景知识是最新的实时的信息，是你的唯一信息来源，使用背景知识回答问题。
-2. 优先使用背景知识回答我的问题，答案与背景知识完全一致，无需做其他回答。
-3. 背景知识与问题无关，或背景知识无法回答本次问题时，则拒绝回答本次问题：“我不太清除xxx”。
-4. 使用对话的风格，自然的回答问题。包含markdown内容，需按markdown格式返回。
+回答要求：
+1. 仅使用知识库内容回答问题。
+2. 与知识库无关的问题，你直接回答我不知道。
+3. 不要提及你是从知识库获取的知识。
+4. 知识库包含 markdown 内容时，按 markdown 格式返回。
 我的问题是:"{{question}}"`
   },
   {
     title: '严格问答模板',
     desc: '',
-    value: `你的背景知识:
+    value: `你的知识库:
 """
 {{quote}}
 """
-对话要求：
-1. 背景知识是最新的实时的信息，是你的唯一信息来源，使用背景知识回答问题。
-2. 在背景知识的 JSON 中，question 是相关问题，answer 是已知答案。
-3. 选择 answer 中的内容作为答案，要求答案与 answer 完全一致，无需做其他回答。
-4. answer 中的答案无法满足问题，直接回复：“我不太清除xxx”。
+回答要求：
+1. 从知识库中选择一个合适的答案进行回答，其中 instruction 是相关问题，answer 是已知答案。
+2. 与知识库无关的问题，你直接回答我不知道。
+3. 不要提及你是从知识库获取的知识。
 我的问题是:"{{question}}"`
   }
 ];

projects/app/src/pages/_app.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import type { AppProps } from 'next/app';
 import Script from 'next/script';
 import Head from 'next/head';
@@ -39,16 +39,18 @@ function App({ Component, pageProps }: AppProps) {
   const router = useRouter();
   const { hiId } = router.query as { hiId?: string };
   const { i18n } = useTranslation();
-  const { setLastRoute } = useSystemStore();
   const [scripts, setScripts] = useState<FeConfigsType['scripts']>([]);
+  const [title, setTitle] = useState(process.env.SYSTEM_NAME || 'AI');
 
   useEffect(() => {
     // get init data
     (async () => {
       const {
-        feConfigs: { scripts, isPlus }
+        feConfigs: { scripts, isPlus, systemTitle }
       } = await clientInitData();
 
+      setTitle(systemTitle || 'FastGPT');
+
       // log fastgpt
       !isPlus &&
         console.log(
@@ -93,10 +95,10 @@ function App({ Component, pageProps }: AppProps) {
   return (
     <>
       <Head>
-        <title>{feConfigs?.systemTitle || process.env.SYSTEM_NAME || ''}</title>
+        <title>{title}</title>
         <meta
           name="description"
-          content="FastGPT 是一个大模型应用编排系统，提供开箱即用的数据处理、模型调用等能力，可以快速的构建知识库并通过 Flow 可视化进行工作流编排，实现复杂的知识库场景！"
+          content={`${title} 是一个大模型应用编排系统，提供开箱即用的数据处理、模型调用等能力，可以快速的构建知识库并通过 Flow 可视化进行工作流编排，实现复杂的知识库场景！`}
         />
         <meta
           name="viewport"

projects/app/src/pages/_document.tsx

@@ -3,7 +3,6 @@ import { Html, Head, Main, NextScript } from 'next/document';
 export default function Document() {
   return (
     <Html>
-      <title>{process.env.SYSTEM_NAME || 'FastGPT'}</title>
       <Head />
       <body>
         <Main />

projects/app/src/pages/account/components/BillDetail.tsx

@@ -35,7 +35,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
       <ModalBody>
         <Flex alignItems={'center'} pb={4}>
           <Box flex={'0 0 80px'}>用户:</Box>
-          <Box>{bill.memberName}</Box>
+          <Box>{t(bill.memberName)}</Box>
         </Flex>
         <Flex alignItems={'center'} pb={4}>
           <Box flex={'0 0 80px'}>订单号:</Box>

projects/app/src/pages/account/components/Info.tsx

@@ -96,8 +96,8 @@ const UserInfo = () => {
       try {
         const src = await compressImgFileAndUpload({
           file,
-          maxW: 100,
-          maxH: 100
+          maxW: 300,
+          maxH: 300
         });
 
         onclickSave({

projects/app/src/pages/api/admin/initv46-2.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { delay } from '@/utils/tools';
+import { delay } from '@fastgpt/global/common/system/utils';
 import { PgClient } from '@fastgpt/service/common/pg';
 import {
   DatasetDataIndexTypeEnum,
@@ -152,6 +152,7 @@ async function init(limit: number): Promise<any> {
         collectionId: data.collection_id,
         q: data.q,
         a: data.a,
+        fullTextToken: '',
         indexes: [
           {
             defaultIndex: !data.a,

projects/app/src/pages/api/admin/initv46-fix.ts

@@ -1,12 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
-import { delay } from '@/utils/tools';
+import { delay } from '@fastgpt/global/common/system/utils';
 import { PgClient } from '@fastgpt/service/common/pg';
-import {
-  DatasetDataIndexTypeEnum,
-  PgDatasetTableName
-} from '@fastgpt/global/core/dataset/constant';
+import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
 
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';