fix: img compress (#546)

* Fix: Update Custom Chat Title Based on chatId

* remove `new:true` option

* Update update.ts

---------

Co-authored-by: Archer <545436317@qq.com>

Dockerfile

@@ -1,7 +1,7 @@
 # Install dependencies only when needed
 FROM node:18.15-alpine AS deps
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-RUN apk add --no-cache libc6-compat && npm install -g pnpm
+RUN apk add libc6-compat && npm install -g pnpm
 WORKDIR /app
 
 ARG name

docSite/content/docs/custom-models/chatglm2-m3e.md

@@ -59,7 +59,7 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
 
 ## 接入 FastGPT
 
-修改 config.json 配置文件，在 VectorModels 中加入 chatglm2 和 M3E 模型：
+修改 config.json 配置文件，在 ChatModels 中加入 chatglm2, 在 VectorModels 中加入 M3E 模型：
 
 ```json
 "ChatModels": [

docSite/content/docs/custom-models/chatglm2.md

@@ -99,7 +99,7 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
 
 ## 接入 FastGPT
 
-修改 config.json 配置文件，在 VectorModels 中加入 chatglm2 模型：
+修改 config.json 配置文件，在 ChatModels 中加入 chatglm2 模型：
 
 ```json
 "ChatModels": [
@@ -107,10 +107,11 @@ Authorization 为 sk-aaabbbcccdddeeefffggghhhiiijjjkkk。model 为刚刚在 One
   {
     "model": "chatglm2",
     "name": "chatglm2",
-    "maxToken": 8000,
-    "price": 0,
-    "quoteMaxToken": 4000,
-    "maxTemperature": 1.2,
+    "maxContext": 4000,
+    "maxResponse": 4000,
+    "quoteMaxToken": 2000,
+    "maxTemperature": 1,
+    "vision": false, 
     "defaultSystemChatPrompt": ""
   }
 ]

docSite/content/docs/development/configuration.md

@@ -26,7 +26,7 @@ weight: 520
     "qaMaxProcess": 15,  // QA 生成最大进程，结合数据库性能和 key 来设置
     "pgHNSWEfSearch": 100  // pg vector 索引参数，越大精度高但速度慢
   },
-  "ChatModels": [
+  "ChatModels": [ // 对话模型
     {
       "model": "gpt-3.5-turbo-1106",
       "name": "GPT35-1106",
@@ -76,7 +76,7 @@ weight: 520
       "defaultSystemChatPrompt": ""
     }
   ],
-  "QAModels": [
+  "QAModels": [ // QA 生成模型
     {
       "model": "gpt-3.5-turbo-16k",
       "name": "GPT35-16k",
@@ -85,14 +85,14 @@ weight: 520
       "price": 0
     }
   ],
-  "CQModels": [
+  "CQModels": [ // 问题分类模型
     {
       "model": "gpt-3.5-turbo-1106",
       "name": "GPT35-1106",
       "maxContext": 16000,
       "maxResponse": 4000,
       "price": 0,
-      "functionCall": true,
+      "functionCall": true, // 是否支持function call， 不支持的模型需要设置为 false，会走提示词生成
       "functionPrompt": ""
     },
     {
@@ -105,7 +105,7 @@ weight: 520
       "functionPrompt": ""
     }
   ],
-  "ExtractModels": [
+  "ExtractModels": [ // 内容提取模型
     {
       "model": "gpt-3.5-turbo-1106",
       "name": "GPT35-1106",
@@ -116,7 +116,7 @@ weight: 520
       "functionPrompt": ""
     }
   ],
-  "QGModels": [
+  "QGModels": [ // 生成下一步指引
     {
       "model": "gpt-3.5-turbo-1106",
       "name": "GPT35-1106",
@@ -125,7 +125,7 @@ weight: 520
       "price": 0
     }
   ],
-  "VectorModels": [
+  "VectorModels": [ // 向量模型
     {
       "model": "text-embedding-ada-002",
       "name": "Embedding-2",

docSite/content/docs/development/openapi/dataset.md

@@ -21,7 +21,9 @@ weight: 563
 curl --location --request POST 'https://fastgpt.run/api/support/wallet/bill/createTrainingBill' \
 --header 'Authorization: Bearer {{apikey}}' \
 --header 'Content-Type: application/json' \
---data-raw ''
+--data-raw '{
+    "name": "可选，自定义订单名称，例如：文档训练-fastgpt.docx"
+}'
 ```
 
 **响应结果**

docSite/content/docs/installation/docker.md

@@ -86,7 +86,7 @@ curl -O https://raw.githubusercontent.com/labring/FastGPT/main/projects/app/data
 
 ## 三、启动容器
 
-修改`docker-compose.yml`中的`OPENAI_BASE_URL`和`CHAT_API_KEY`即可，对应为 API 的地址和 key。
+修改`docker-compose.yml`中的`OPENAI_BASE_URL`和`CHAT_API_KEY`即可，对应为 API 的地址(别忘记加/v1)和 key。
 
 ```bash
 # 在 docker-compose.yml 同级目录下执行

docSite/content/docs/installation/upgrading/462.md

@@ -0,0 +1,31 @@
+---
+title: 'V4.6.2(需要初始化)'
+description: 'FastGPT V4.6.2'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 834
+---
+
+## 1。执行初始化 API
+
+发起 1 个 HTTP 请求 ({{rootkey}} 替换成环境变量里的 `rootkey`，{{host}} 替换成自己域名)
+
+1. https://xxxxx/api/admin/initv462
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv462' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化说明：
+1. 初始化全文索引
+
+## V4.6.2 功能介绍
+
+1. 新增 - 全文索引（需配合 Rerank 模型，在看怎么放到开源版，模型接口比较特殊）
+2. 新增 - 插件来源（预计4.7/4.8版本会正式使用）
+3. 优化 - PDF读取
+4. 优化 - docx文件读取，转成 markdown 并保留其图片内容
+5. 修复和优化 TextSplitter 函数

docSite/content/docs/installation/upgrading/463.md

@@ -0,0 +1,33 @@
+---
+title: 'V4.6.3(需要初始化)'
+description: 'FastGPT V4.6.3'
+icon: 'upgrade'
+draft: false
+toc: true
+weight: 833
+---
+
+## 1。执行初始化 API
+
+发起 1 个 HTTP 请求 ({{rootkey}} 替换成环境变量里的 `rootkey`，{{host}} 替换成自己域名)
+
+1. https://xxxxx/api/admin/initv463
+
+```bash
+curl --location --request POST 'https://{{host}}/api/admin/initv463' \
+--header 'rootkey: {{rootkey}}' \
+--header 'Content-Type: application/json'
+```
+
+初始化说明：
+1. 初始化Mongo 中 dataset，collection 和 data 的部分字段
+
+## V4.6.3 功能介绍
+
+1. 商业版新增 - web站点同步
+2. 新增 - 集合元数据记录
+3. 优化 - url 读取内容
+4. 优化 - 流读取文件，防止内存溢出
+5. 优化 - 4v模型自动将 url 转 base64，本地也可调试
+6. 优化 - 图片压缩等级
+7. 修复 - 图片压缩失败报错，防止文件读取过程卡死。

packages/global/common/file/api.d.ts

@@ -0,0 +1,8 @@
+export type UrlFetchParams = {
+  urlList: string[];
+  selector?: string;
+};
+export type UrlFetchResponse = {
+  url: string;
+  content: string;
+}[];

packages/global/common/file/tools.ts

@@ -1,3 +1,8 @@
+import axios from 'axios';
+import { UrlFetchParams, UrlFetchResponse } from './api.d';
+import { htmlToMarkdown } from '../string/markdown';
+import * as cheerio from 'cheerio';
+
 export const formatFileSize = (bytes: number): string => {
   if (bytes === 0) return '0 B';
 
@@ -7,3 +12,84 @@ export const formatFileSize = (bytes: number): string => {
 
   return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
 };
+
+export const cheerioToHtml = ({
+  fetchUrl,
+  $,
+  selector
+}: {
+  fetchUrl: string;
+  $: cheerio.CheerioAPI;
+  selector?: string;
+}) => {
+  // get origin url
+  const originUrl = new URL(fetchUrl).origin;
+
+  // remove i element
+  $('i,script').remove();
+
+  // remove empty a element
+  $('a')
+    .filter((i, el) => {
+      return $(el).text().trim() === '' && $(el).children().length === 0;
+    })
+    .remove();
+
+  // if link,img startWith /, add origin url
+  $('a').each((i, el) => {
+    const href = $(el).attr('href');
+    if (href && href.startsWith('/')) {
+      $(el).attr('href', originUrl + href);
+    }
+  });
+  $('img').each((i, el) => {
+    const src = $(el).attr('src');
+    if (src && src.startsWith('/')) {
+      $(el).attr('src', originUrl + src);
+    }
+  });
+
+  return $(selector || 'body').html();
+};
+export const urlsFetch = async ({
+  urlList,
+  selector
+}: UrlFetchParams): Promise<UrlFetchResponse> => {
+  urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
+
+  const response = (
+    await Promise.all(
+      urlList.map(async (url) => {
+        try {
+          const fetchRes = await axios.get(url, {
+            timeout: 30000
+          });
+
+          const $ = cheerio.load(fetchRes.data);
+
+          const md = htmlToMarkdown(
+            cheerioToHtml({
+              fetchUrl: url,
+              $,
+              selector
+            })
+          );
+
+          return {
+            url,
+            content: md
+          };
+        } catch (error) {
+          console.log(error, 'fetch error');
+
+          return {
+            url,
+            content: ''
+          };
+        }
+      })
+    )
+  ).filter((item) => item.content);
+
+  return response;
+};

packages/global/common/plugin/types/pluginRes.d.ts

@@ -1,4 +0,0 @@
-export type FetchResultItem = {
-  url: string;
-  content: string;
-};

packages/global/common/string/markdown.ts

@@ -0,0 +1,97 @@
+import { simpleText } from './tools';
+import { NodeHtmlMarkdown } from 'node-html-markdown';
+
+/* Delete redundant text in markdown */
+export const simpleMarkdownText = (rawText: string) => {
+  rawText = simpleText(rawText);
+
+  // Remove a line feed from a hyperlink or picture
+  rawText = rawText.replace(/\[([^\]]+)\]\((.+?)\)/g, (match, linkText, url) => {
+    const cleanedLinkText = linkText.replace(/\n/g, ' ').trim();
+
+    if (!url) {
+      return '';
+    }
+
+    return `[${cleanedLinkText}](${url})`;
+  });
+
+  // replace special \.* ……
+  const reg1 = /\\([-.!`_(){}\[\]])/g;
+  if (reg1.test(rawText)) {
+    rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1');
+  }
+
+  // replace \\n
+  rawText = rawText.replace(/\\\\n/g, '\\n');
+
+  // Remove headings and code blocks front spaces
+  ['####', '###', '##', '#', '```', '~~~'].forEach((item) => {
+    const reg = new RegExp(`\\n\\s*${item}`, 'g');
+    if (reg.test(rawText)) {
+      rawText = rawText.replace(new RegExp(`\\n\\s*(${item})`, 'g'), '\n$1');
+    }
+  });
+
+  return rawText.trim();
+};
+
+/* html string to markdown */
+export const htmlToMarkdown = (html?: string | null) => {
+  if (!html) return '';
+
+  const surround = (source: string, surroundStr: string) => `${surroundStr}${source}${surroundStr}`;
+
+  const nhm = new NodeHtmlMarkdown(
+    {
+      codeFence: '```',
+      codeBlockStyle: 'fenced',
+      ignore: ['i', 'script']
+    },
+    {
+      code: ({ node, parent, options: { codeFence, codeBlockStyle }, visitor }) => {
+        const isCodeBlock = ['PRE', 'WRAPPED-PRE'].includes(parent?.tagName!);
+
+        if (!isCodeBlock) {
+          return {
+            spaceIfRepeatingChar: true,
+            noEscape: true,
+            postprocess: ({ content }) => {
+              // Find longest occurring sequence of running backticks and add one more (so content is escaped)
+              const delimiter =
+                '`' + (content.match(/`+/g)?.sort((a, b) => b.length - a.length)?.[0] || '');
+              const padding = delimiter.length > 1 ? ' ' : '';
+
+              return surround(surround(content, padding), delimiter);
+            }
+          };
+        }
+
+        /* Handle code block */
+        if (codeBlockStyle === 'fenced') {
+          const language =
+            node.getAttribute('class')?.match(/language-(\S+)/)?.[1] ||
+            parent?.getAttribute('class')?.match(/language-(\S+)/)?.[1] ||
+            '';
+
+          return {
+            noEscape: true,
+            prefix: `${codeFence}${language}\n`,
+            postfix: `\n${codeFence}\n`,
+            childTranslators: visitor.instance.codeBlockTranslators
+          };
+        }
+
+        return {
+          noEscape: true,
+          postprocess: ({ content }) => content.replace(/^/gm, '    '),
+          childTranslators: visitor.instance.codeBlockTranslators
+        };
+      }
+    }
+  );
+
+  const markdown = nhm.translate(html).trim();
+
+  return simpleMarkdownText(markdown);
+};

packages/global/common/string/textSplitter.ts

@@ -3,121 +3,218 @@ import { countPromptTokens } from './tiktoken';
 
 /**
  * text split into chunks
- * maxLen - one chunk len. max: 3500
+ * chunkLen - one chunk len. max: 3500
  * overlapLen - The size of the before and after Text
- * maxLen > overlapLen
+ * chunkLen > overlapLen
  * markdown
  */
-export const splitText2Chunks = (props: { text: string; maxLen: number; overlapLen?: number }) => {
-  const { text = '', maxLen, overlapLen = Math.floor(maxLen * 0.2) } = props;
-  const tempMarker = 'SPLIT_HERE_SPLIT_HERE';
+export const splitText2Chunks = (props: {
+  text: string;
+  chunkLen: number;
+  overlapRatio?: number;
+}): {
+  chunks: string[];
+  tokens: number;
+  overlapRatio?: number;
+} => {
+  let { text = '', chunkLen, overlapRatio = 0.2 } = props;
+  const splitMarker = 'SPLIT_HERE_SPLIT_HERE';
+  const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER';
+  const overlapLen = Math.round(chunkLen * overlapRatio);
 
-  const stepReg: Record<number, RegExp> = {
-    0: /^(#\s[^\n]+)\n/gm,
-    1: /^(##\s[^\n]+)\n/gm,
-    2: /^(###\s[^\n]+)\n/gm,
-    3: /^(####\s[^\n]+)\n/gm,
+  // replace code block all \n to codeBlockMarker
+  text = text.replace(/(```[\s\S]*?```|~~~[\s\S]*?~~~)/g, function (match) {
+    return match.replace(/\n/g, codeBlockMarker);
+  });
 
-    4: /(\n\n)/g,
-    5: /([\n])/g,
-    6: /[。]|(?!<[^a-zA-Z])\.\s/g,
-    7: /([！？]|!\s|\?\s)/g,
-    8: /([；]|;\s)/g,
-    9: /([，]|,\s)/g
+  // The larger maxLen is, the next sentence is less likely to trigger splitting
+  const stepReges: { reg: RegExp; maxLen: number }[] = [
+    { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
+    { reg: /^(##\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
+    { reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
+    { reg: /^(####\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 },
+
+    { reg: /([\n](`))/g, maxLen: chunkLen * 4 }, // code block
+    { reg: /([\n](?![\*\-|>0-9]))/g, maxLen: chunkLen * 1.8 }, // (?![\*\-|>`0-9]): markdown special char
+    { reg: /([\n])/g, maxLen: chunkLen * 1.4 },
+
+    { reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.4 },
+    { reg: /([！]|!\s)/g, maxLen: chunkLen * 1.4 },
+    { reg: /([？]|\?\s)/g, maxLen: chunkLen * 1.6 },
+    { reg: /([；]|;\s)/g, maxLen: chunkLen * 1.8 },
+    { reg: /([，]|,\s)/g, maxLen: chunkLen * 2 }
+  ];
+
+  // if use markdown title split, Separate record title title
+  const getSplitTexts = ({ text, step }: { text: string; step: number }) => {
+    if (step >= stepReges.length) {
+      return [
+        {
+          text,
+          title: ''
+        }
+      ];
+    }
+    const isMarkdownSplit = step <= 3;
+    const { reg } = stepReges[step];
+
+    const splitTexts = text
+      .replace(reg, isMarkdownSplit ? `${splitMarker}$1` : `$1${splitMarker}`)
+      .split(`${splitMarker}`)
+      .filter((part) => part.trim());
+
+    return splitTexts
+      .map((text) => {
+        const matchTitle = isMarkdownSplit ? text.match(reg)?.[0] || '' : '';
+
+        return {
+          text: isMarkdownSplit ? text.replace(matchTitle, '') : text,
+          title: matchTitle
+        };
+      })
+      .filter((item) => item.text.trim());
+  };
+
+  const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => {
+    const forbidOverlap = step <= 6;
+    const maxOverlapLen = chunkLen * 0.4;
+
+    // step >= stepReges.length: Do not overlap incomplete sentences
+    if (forbidOverlap || overlapLen === 0 || step >= stepReges.length) return '';
+
+    const splitTexts = getSplitTexts({ text, step });
+    let overlayText = '';
+
+    for (let i = splitTexts.length - 1; i >= 0; i--) {
+      const currentText = splitTexts[i].text;
+      const newText = currentText + overlayText;
+      const newTextLen = newText.length;
+
+      if (newTextLen > overlapLen) {
+        if (newTextLen > maxOverlapLen) {
+          const text = getOneTextOverlapText({ text: newText, step: step + 1 });
+          return text || overlayText;
+        }
+        return newText;
+      }
+
+      overlayText = newText;
+    }
+    return overlayText;
   };
 
   const splitTextRecursively = ({
     text = '',
     step,
-    lastChunk,
-    overlayChunk
+    lastText,
+    mdTitle = ''
   }: {
     text: string;
     step: number;
-    lastChunk: string;
-    overlayChunk: string;
-  }) => {
-    if (text.length <= maxLen) {
+    lastText: string;
+    mdTitle: string;
+  }): string[] => {
+    const isMarkdownSplit = step <= 3;
+
+    // mini text
+    if (text.length <= chunkLen) {
       return [text];
     }
-    const reg = stepReg[step];
-    const isMarkdownSplit = step < 4;
 
-    if (!reg) {
-      // use slice-maxLen to split text
+    // oversize
+    if (step >= stepReges.length) {
+      if (text.length < chunkLen * 3) {
+        return [text];
+      }
+      // use slice-chunkLen to split text
       const chunks: string[] = [];
-      let chunk = '';
-      for (let i = 0; i < text.length; i += maxLen - overlapLen) {
-        chunk = text.slice(i, i + maxLen);
-        chunks.push(chunk);
+      for (let i = 0; i < text.length; i += chunkLen - overlapLen) {
+        chunks.push(`${mdTitle}${text.slice(i, i + chunkLen)}`);
       }
       return chunks;
     }
 
+    const { maxLen } = stepReges[step];
+    const minChunkLen = chunkLen * 0.7;
+
     // split text by special char
-    const splitTexts = text
-      .replace(reg, isMarkdownSplit ? `${tempMarker}$1` : `$1${tempMarker}`)
-      .split(`${tempMarker}`)
-      .filter((part) => part);
+    const splitTexts = getSplitTexts({ text, step });
 
-    let chunks: string[] = [];
+    const chunks: string[] = [];
     for (let i = 0; i < splitTexts.length; i++) {
-      let text = splitTexts[i];
-      let chunkToken = lastChunk.length;
-      const textToken = text.length;
+      const item = splitTexts[i];
+      const currentTitle = `${mdTitle}${item.title}`;
 
-      // next chunk is too large / new chunk is too large(The current chunk must be smaller than maxLen)
-      if (textToken >= maxLen || chunkToken + textToken > maxLen * 1.4) {
-        // last chunk is too large, push it to chunks, not add to next chunk
-        if (chunkToken > maxLen * 0.7) {
-          chunks.push(lastChunk);
-          lastChunk = '';
-          overlayChunk = '';
+      const currentText = item.text;
+      const currentTextLen = currentText.length;
+      const lastTextLen = lastText.length;
+      const newText = lastText + currentText;
+      const newTextLen = lastTextLen + currentTextLen;
+
+      // newText is too large(now, The lastText must be smaller than chunkLen)
+      if (newTextLen > maxLen) {
+        // lastText greater minChunkLen, direct push it to chunks, not add to next chunk. (large lastText)
+        if (lastTextLen > minChunkLen) {
+          chunks.push(`${currentTitle}${lastText}`);
+          lastText = getOneTextOverlapText({ text: lastText, step }); // next chunk will start with overlayText
+          i--;
+
+          continue;
         }
-        // chunk is small, insert to next chunks
+
+        // split new Text, split chunks must will greater 1 (small lastText)
         const innerChunks = splitTextRecursively({
-          text,
+          text: newText,
           step: step + 1,
-          lastChunk,
-          overlayChunk
+          lastText: '',
+          mdTitle: currentTitle
         });
-        if (innerChunks.length === 0) continue;
-        chunks = chunks.concat(innerChunks);
-        lastChunk = '';
-        overlayChunk = '';
+        const lastChunk = innerChunks[innerChunks.length - 1];
+        // last chunk is too small, concat it to lastText
+        if (!isMarkdownSplit && lastChunk.length < minChunkLen) {
+          chunks.push(...innerChunks.slice(0, -1));
+          lastText = lastChunk;
+        } else {
+          chunks.push(...innerChunks);
+          // compute new overlapText
+          lastText = getOneTextOverlapText({
+            text: lastChunk,
+            step
+          });
+        }
         continue;
       }
 
-      // size less than maxLen, push text to last chunk
-      lastChunk += text;
-      chunkToken += textToken; // Definitely less than 1.4 * maxLen
+      // size less than chunkLen, push text to last chunk. now, text definitely less than maxLen
+      lastText = newText;
 
-      // size over lapLen, push it to next chunk
-      if (
-        overlapLen !== 0 &&
-        !isMarkdownSplit &&
-        chunkToken >= maxLen - overlapLen &&
-        textToken < overlapLen
-      ) {
-        overlayChunk += text;
-      }
-      if (chunkToken >= maxLen) {
-        chunks.push(lastChunk);
-        lastChunk = overlayChunk;
-        overlayChunk = '';
+      // markdown paragraph block: Direct addition; If the chunk size reaches, add a chunk
+      if (isMarkdownSplit || newTextLen >= chunkLen) {
+        chunks.push(`${currentTitle}${lastText}`);
+
+        lastText = isMarkdownSplit ? '' : getOneTextOverlapText({ text: lastText, step });
       }
     }
 
     /* If the last chunk is independent, it needs to be push chunks. */
-    if (lastChunk && chunks[chunks.length - 1] && !chunks[chunks.length - 1].endsWith(lastChunk)) {
-      chunks.push(lastChunk);
+    if (lastText && chunks[chunks.length - 1] && !chunks[chunks.length - 1].endsWith(lastText)) {
+      if (lastText.length < chunkLen * 0.4) {
+        chunks[chunks.length - 1] = chunks[chunks.length - 1] + lastText;
+      } else {
+        chunks.push(`${mdTitle}${lastText}`);
+      }
     }
 
     return chunks;
   };
 
   try {
-    const chunks = splitTextRecursively({ text, step: 0, lastChunk: '', overlayChunk: '' });
+    const chunks = splitTextRecursively({
+      text,
+      step: 0,
+      lastText: '',
+      mdTitle: ''
+    }).map((chunk) => chunk.replaceAll(codeBlockMarker, '\n')); // restore code block
 
     const tokens = chunks.reduce((sum, chunk) => sum + countPromptTokens(chunk, 'system'), 0);
 

packages/global/common/string/time.ts

@@ -0,0 +1,3 @@
+import dayjs from 'dayjs';
+
+export const formatTime2YMDHM = (time: Date) => dayjs(time).format('YYYY-MM-DD HH:mm');

packages/global/common/string/tools.ts

@@ -13,7 +13,8 @@ export const hashStr = (str: string) => {
 };
 
 /* simple text, remove chinese space and extra \n */
-export const simpleText = (text: string) => {
+export const simpleText = (text = '') => {
+  text = text.trim();
   text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
   text = text.replace(/\r\n|\r/g, '\n');
   text = text.replace(/\n{3,}/g, '\n\n');

packages/global/common/system/types/index.d.ts

@@ -2,16 +2,15 @@ export type FeConfigsType = {
   show_emptyChat?: boolean;
   show_register?: boolean;
   show_appStore?: boolean;
-  show_contact?: boolean;
   show_git?: boolean;
   show_pay?: boolean;
   show_openai_account?: boolean;
   show_promotion?: boolean;
   hide_app_flow?: boolean;
+  concatMd?: string;
   docUrl?: string;
   openAPIDocUrl?: string;
   systemTitle?: string;
-  authorText?: string;
   googleClientVerKey?: string;
   isPlus?: boolean;
   oauth?: {

packages/global/common/system/utils.ts

@@ -0,0 +1,6 @@
+export const delay = (ms: number) =>
+  new Promise((resolve) => {
+    setTimeout(() => {
+      resolve('');
+    }, ms);
+  });

packages/global/core/ai/api.d.ts

@@ -2,4 +2,4 @@ export type PostReRankProps = {
   query: string;
   inputs: { id: string; text: string }[];
 };
-export type PostReRankResponse = { id: string; score: number }[];
+export type PostReRankResponse = { id: string; score?: number }[];

packages/global/core/ai/model.d.ts

@@ -26,6 +26,14 @@ export type VectorModelItemType = {
   maxToken: number;
 };
 
+export type ReRankModelItemType = {
+  model: string;
+  name: string;
+  price: number;
+  requestUrl?: string;
+  requestAuth?: string;
+};
+
 export type AudioSpeechModelType = {
   model: string;
   name: string;

packages/global/core/ai/model.ts

@@ -4,7 +4,8 @@ import type {
   FunctionModelItemType,
   VectorModelItemType,
   AudioSpeechModelType,
-  WhisperModelType
+  WhisperModelType,
+  ReRankModelItemType
 } from './model.d';
 
 export const defaultChatModels: ChatModelItemType[] = [
@@ -117,6 +118,8 @@ export const defaultVectorModels: VectorModelItemType[] = [
   }
 ];
 
+export const defaultReRankModels: ReRankModelItemType[] = [];
+
 export const defaultAudioSpeechModels: AudioSpeechModelType[] = [
   {
     model: 'tts-1',

packages/global/core/app/api.d.ts

@@ -1,5 +1,6 @@
+import type { ChatModelItemType } from '../ai/model.d';
 import { AppTypeEnum } from './constants';
-import { AppSchema } from './type';
+import { AppSchema, AppSimpleEditFormType } from './type';
 
 export type CreateAppParams = {
   name?: string;
@@ -11,8 +12,15 @@ export type CreateAppParams = {
 export interface AppUpdateParams {
   name?: string;
   type?: `${AppTypeEnum}`;
+  simpleTemplateId?: string;
   avatar?: string;
   intro?: string;
   modules?: AppSchema['modules'];
   permission?: AppSchema['permission'];
 }
+
+export type FormatForm2ModulesProps = {
+  formData: AppSimpleEditFormType;
+  chatModelMaxToken: number;
+  chatModelList: ChatModelItemType[];
+};

packages/global/core/app/constants.ts

@@ -1,4 +1,12 @@
 export enum AppTypeEnum {
-  basic = 'basic',
+  simple = 'simple',
   advanced = 'advanced'
 }
+export const AppTypeMap = {
+  [AppTypeEnum.simple]: {
+    label: 'simple'
+  },
+  [AppTypeEnum.advanced]: {
+    label: 'advanced'
+  }
+};

packages/global/core/app/type.d.ts

@@ -1,6 +1,10 @@
-import { ModuleItemType } from '../module/type';
+import type { AppTTSConfigType, ModuleItemType, VariableItemType } from '../module/type.d';
 import { AppTypeEnum } from './constants';
 import { PermissionTypeEnum } from '../../support/permission/constant';
+import type { AIChatModuleProps, DatasetModuleProps } from '../module/node/type.d';
+import { VariableInputEnum } from '../module/constants';
+import { SelectedDatasetType } from '../module/api';
+import { DatasetSearchModeEnum } from '../dataset/constant';
 
 export interface AppSchema {
   _id: string;
@@ -9,11 +13,13 @@ export interface AppSchema {
   tmbId: string;
   name: string;
   type: `${AppTypeEnum}`;
+  simpleTemplateId: string;
   avatar: string;
   intro: string;
   updateTime: number;
   modules: ModuleItemType[];
   permission: `${PermissionTypeEnum}`;
+  inited?: boolean;
 }
 
 export type AppListItemType = {
@@ -29,3 +35,87 @@ export type AppDetailType = AppSchema & {
   isOwner: boolean;
   canWrite: boolean;
 };
+
+// export type AppSimpleEditFormType = {
+//   aiSettings: AIChatModuleProps;
+//   dataset: DatasetModuleProps & {
+//     searchEmptyText: string;
+//   };
+//   userGuide: {
+//     welcomeText: string;
+//     variables: VariableItemType[];
+//     questionGuide: boolean;
+//     tts: AppTTSConfigType;
+//   };
+// };
+// Since useform cannot infer enumeration types, all enumeration keys can only be undone manually
+export type AppSimpleEditFormType = {
+  templateId: string;
+  aiSettings: {
+    model: string;
+    systemPrompt?: string | undefined;
+    temperature: number;
+    maxToken: number;
+    isResponseAnswerText: boolean;
+    quoteTemplate?: string | undefined;
+    quotePrompt?: string | undefined;
+  };
+  dataset: {
+    datasets: SelectedDatasetType;
+    similarity: number;
+    limit: number;
+    searchMode: `${DatasetSearchModeEnum}`;
+    searchEmptyText: string;
+  };
+  userGuide: {
+    welcomeText: string;
+    variables: {
+      id: string;
+      key: string;
+      label: string;
+      type: `${VariableInputEnum}`;
+      required: boolean;
+      maxLen: number;
+      enums: {
+        value: string;
+      }[];
+    }[];
+    questionGuide: boolean;
+    tts: {
+      type: 'none' | 'web' | 'model';
+      model?: string | undefined;
+      voice?: string | undefined;
+      speed?: number | undefined;
+    };
+  };
+};
+
+/* simple mode template*/
+export type AppSimpleEditConfigTemplateType = {
+  id: string;
+  name: string;
+  desc: string;
+  systemForm: {
+    aiSettings?: {
+      model?: boolean;
+      systemPrompt?: boolean;
+      temperature?: boolean;
+      maxToken?: boolean;
+      quoteTemplate?: boolean;
+      quotePrompt?: boolean;
+    };
+    dataset?: {
+      datasets?: boolean;
+      similarity?: boolean;
+      limit?: boolean;
+      searchMode: `${DatasetSearchModeEnum}`;
+      searchEmptyText?: boolean;
+    };
+    userGuide?: {
+      welcomeText?: boolean;
+      variables?: boolean;
+      questionGuide?: boolean;
+      tts?: boolean;
+    };
+  };
+};

packages/global/core/app/utils.ts

@@ -0,0 +1,123 @@
+import type { AppSimpleEditFormType } from '../app/type';
+import { FlowNodeTypeEnum } from '../module/node/constant';
+import { ModuleOutputKeyEnum, ModuleInputKeyEnum } from '../module/constants';
+import type { FlowNodeInputItemType } from '../module/node/type.d';
+import { getGuideModule, splitGuideModule } from '../module/utils';
+import { defaultChatModels } from '../ai/model';
+import { ModuleItemType } from '../module/type.d';
+import { DatasetSearchModeEnum } from '../dataset/constant';
+
+export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEditFormType => {
+  const defaultChatModel = defaultChatModels[0];
+
+  return {
+    templateId,
+    aiSettings: {
+      model: defaultChatModel?.model,
+      systemPrompt: '',
+      temperature: 0,
+      isResponseAnswerText: true,
+      quotePrompt: '',
+      quoteTemplate: '',
+      maxToken: defaultChatModel ? defaultChatModel.maxResponse / 2 : 4000
+    },
+    dataset: {
+      datasets: [],
+      similarity: 0.4,
+      limit: 5,
+      searchEmptyText: '',
+      searchMode: DatasetSearchModeEnum.embedding
+    },
+    userGuide: {
+      welcomeText: '',
+      variables: [],
+      questionGuide: false,
+      tts: {
+        type: 'web'
+      }
+    }
+  };
+};
+
+/* format app modules to edit form */
+export const appModules2Form = ({
+  templateId,
+  modules
+}: {
+  modules: ModuleItemType[];
+  templateId: string;
+}) => {
+  const defaultAppForm = getDefaultAppForm(templateId);
+
+  const findInputValueByKey = (inputs: FlowNodeInputItemType[], key: string) => {
+    return inputs.find((item) => item.key === key)?.value;
+  };
+
+  modules.forEach((module) => {
+    if (module.flowType === FlowNodeTypeEnum.chatNode) {
+      defaultAppForm.aiSettings.model = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.aiModel
+      );
+      defaultAppForm.aiSettings.systemPrompt = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.aiSystemPrompt
+      );
+      defaultAppForm.aiSettings.temperature = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.aiChatTemperature
+      );
+      defaultAppForm.aiSettings.maxToken = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.aiChatMaxToken
+      );
+      defaultAppForm.aiSettings.quoteTemplate = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.aiChatQuoteTemplate
+      );
+      defaultAppForm.aiSettings.quotePrompt = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.aiChatQuotePrompt
+      );
+    } else if (module.flowType === FlowNodeTypeEnum.datasetSearchNode) {
+      defaultAppForm.dataset.datasets = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.datasetSelectList
+      );
+      defaultAppForm.dataset.similarity = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.datasetSimilarity
+      );
+      defaultAppForm.dataset.limit = findInputValueByKey(
+        module.inputs,
+        ModuleInputKeyEnum.datasetLimit
+      );
+      defaultAppForm.dataset.searchMode =
+        findInputValueByKey(module.inputs, ModuleInputKeyEnum.datasetSearchMode) ||
+        DatasetSearchModeEnum.embedding;
+
+      // empty text
+      const emptyOutputs =
+        module.outputs.find((item) => item.key === ModuleOutputKeyEnum.datasetIsEmpty)?.targets ||
+        [];
+      const emptyOutput = emptyOutputs[0];
+      if (emptyOutput) {
+        const target = modules.find((item) => item.moduleId === emptyOutput.moduleId);
+        defaultAppForm.dataset.searchEmptyText =
+          target?.inputs?.find((item) => item.key === ModuleInputKeyEnum.answerText)?.value || '';
+      }
+    } else if (module.flowType === FlowNodeTypeEnum.userGuide) {
+      const { welcomeText, variableModules, questionGuide, ttsConfig } = splitGuideModule(
+        getGuideModule(modules)
+      );
+      defaultAppForm.userGuide = {
+        welcomeText: welcomeText,
+        variables: variableModules,
+        questionGuide: questionGuide,
+        tts: ttsConfig
+      };
+    }
+  });
+
+  return defaultAppForm;
+};

packages/global/core/chat/api.d.ts

@@ -30,5 +30,4 @@ export type InitChatResponse = {
 export type ChatHistoryItemResType = moduleDispatchResType & {
   moduleType: `${FlowNodeTypeEnum}`;
   moduleName: string;
-  moduleLogo?: string;
 };

packages/global/core/chat/constants.ts

@@ -5,13 +5,6 @@ export enum ChatRoleEnum {
   Function = 'Function',
   Tool = 'Tool'
 }
-
-export enum TaskResponseKeyEnum {
-  'answerText' = 'answerText', //  answer module text key
-  'responseData' = 'responseData',
-  'history' = 'history'
-}
-
 export const ChatRoleMap = {
   [ChatRoleEnum.System]: {
     name: '系统提示词'
@@ -36,7 +29,6 @@ export enum ChatSourceEnum {
   share = 'share',
   api = 'api'
 }
-
 export const ChatSourceMap = {
   [ChatSourceEnum.test]: {
     name: 'chat.logs.test'

packages/global/core/chat/type.d.ts

@@ -1,8 +1,10 @@
 import { ClassifyQuestionAgentItemType } from '../module/type';
 import { SearchDataResponseItemType } from '../dataset/type';
-import { ChatRoleEnum, ChatSourceEnum, TaskResponseKeyEnum } from './constants';
+import { ChatRoleEnum, ChatSourceEnum } from './constants';
 import { FlowNodeTypeEnum } from '../module/node/constant';
-import { AppSchema } from 'core/app/type';
+import { ModuleOutputKeyEnum } from '../module/constants';
+import { AppSchema } from '../app/type';
+import { DatasetSearchModeEnum } from '../dataset/constant';
 
 export type ChatSchema = {
   _id: string;
@@ -38,7 +40,7 @@ export type ChatItemSchema = {
   value: string;
   userFeedback?: string;
   adminFeedback?: AdminFbkType;
-  [TaskResponseKeyEnum.responseData]?: ChatHistoryItemResType[];
+  [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };
 
 export type AdminFbkType = {
@@ -55,14 +57,14 @@ export type ChatItemType = {
   value: any;
   userFeedback?: string;
   adminFeedback?: ChatItemSchema['feedback'];
-  [TaskResponseKeyEnum.responseData]?: ChatItemSchema[TaskResponseKeyEnum.responseData];
+  [ModuleOutputKeyEnum.responseData]?: ChatHistoryItemResType[];
 };
 
-export type ChatSiteItemType = {
+export type ChatSiteItemType = ChatItemType & {
   status: 'loading' | 'running' | 'finish';
   moduleName?: string;
   ttsBuffer?: Uint8Array;
-} & ChatItemType;
+};
 
 export type HistoryItemType = {
   chatId: string;
@@ -77,13 +79,14 @@ export type ChatHistoryItemType = HistoryItemType & {
 
 // response data
 export type moduleDispatchResType = {
+  moduleLogo?: string;
   price: number;
   runningTime?: number;
   tokens?: number;
   model?: string;
+  query?: string;
 
   // chat
-  question?: string;
   temperature?: number;
   maxToken?: number;
   quoteList?: SearchDataResponseItemType[];
@@ -92,6 +95,7 @@ export type moduleDispatchResType = {
   // dataset search
   similarity?: number;
   limit?: number;
+  searchMode?: `${DatasetSearchModeEnum}`;
 
   // cq
   cqList?: ClassifyQuestionAgentItemType[];

packages/global/core/dataset/api.d.ts

@@ -1,8 +1,32 @@
-import { DatasetDataIndexItemType } from './type';
+import { DatasetDataIndexItemType, DatasetSchemaType } from './type';
+import { DatasetCollectionTrainingModeEnum, DatasetCollectionTypeEnum } from './constant';
+import type { LLMModelItemType } from '../ai/model.d';
 
 /* ================= dataset ===================== */
+export type DatasetUpdateBody = {
+  id: string;
+  parentId?: string;
+  tags?: string[];
+  name?: string;
+  avatar?: string;
+  permission?: DatasetSchemaType['permission'];
+  agentModel?: LLMModelItemType;
+  websiteConfig?: DatasetSchemaType['websiteConfig'];
+  status?: DatasetSchemaType['status'];
+};
 
 /* ================= collection ===================== */
+export type CreateDatasetCollectionParams = {
+  datasetId: string;
+  parentId?: string;
+  name: string;
+  type: `${DatasetCollectionTypeEnum}`;
+  trainingType?: `${DatasetCollectionTrainingModeEnum}`;
+  chunkSize?: number;
+  fileId?: string;
+  rawLink?: string;
+  metadata?: Record<string, any>;
+};
 
 /* ================= data ===================== */
 export type PgSearchRawType = {
@@ -18,3 +42,8 @@ export type PushDatasetDataChunkProps = {
   a?: string; // bonus content
   indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];
 };
+
+export type PostWebsiteSyncParams = {
+  datasetId: string;
+  billId: string;
+};

packages/global/core/dataset/constant.ts

@@ -1,41 +1,81 @@
 export const PgDatasetTableName = 'modeldata';
 
+/* ------------ dataset -------------- */
 export enum DatasetTypeEnum {
   folder = 'folder',
-  dataset = 'dataset'
+  dataset = 'dataset',
+  websiteDataset = 'websiteDataset' // depp link
 }
-
 export const DatasetTypeMap = {
   [DatasetTypeEnum.folder]: {
-    name: 'folder'
+    icon: 'core/dataset/folderDataset',
+    label: 'core.dataset.Folder Dataset',
+    collectionLabel: 'common.Folder'
   },
   [DatasetTypeEnum.dataset]: {
-    name: 'dataset'
+    icon: 'core/dataset/commonDataset',
+    label: 'core.dataset.Common Dataset',
+    collectionLabel: 'common.File'
+  },
+  [DatasetTypeEnum.websiteDataset]: {
+    icon: 'core/dataset/websiteDataset',
+    label: 'core.dataset.Website Dataset',
+    collectionLabel: 'common.Website'
   }
 };
 
+export enum DatasetStatusEnum {
+  active = 'active',
+  syncing = 'syncing'
+}
+export const DatasetStatusMap = {
+  [DatasetStatusEnum.active]: {
+    label: 'core.dataset.status.active'
+  },
+  [DatasetStatusEnum.syncing]: {
+    label: 'core.dataset.status.syncing'
+  }
+};
+
+/* ------------ collection -------------- */
 export enum DatasetCollectionTypeEnum {
-  file = 'file',
   folder = 'folder',
-  link = 'link',
+  file = 'file',
+  link = 'link', // one link
   virtual = 'virtual'
 }
-
 export const DatasetCollectionTypeMap = {
-  [DatasetCollectionTypeEnum.file]: {
-    name: 'dataset.file'
-  },
   [DatasetCollectionTypeEnum.folder]: {
-    name: 'dataset.folder'
+    name: 'core.dataset.folder'
+  },
+  [DatasetCollectionTypeEnum.file]: {
+    name: 'core.dataset.file'
   },
   [DatasetCollectionTypeEnum.link]: {
-    name: 'dataset.link'
+    name: 'core.dataset.link'
   },
   [DatasetCollectionTypeEnum.virtual]: {
-    name: 'dataset.Virtual File'
+    name: 'core.dataset.Virtual File'
+  }
+};
+export enum DatasetCollectionTrainingModeEnum {
+  manual = 'manual',
+  chunk = 'chunk',
+  qa = 'qa'
+}
+export const DatasetCollectionTrainingTypeMap = {
+  [DatasetCollectionTrainingModeEnum.manual]: {
+    label: 'core.dataset.collection.training.type manual'
+  },
+  [DatasetCollectionTrainingModeEnum.chunk]: {
+    label: 'core.dataset.collection.training.type chunk'
+  },
+  [DatasetCollectionTrainingModeEnum.qa]: {
+    label: 'core.dataset.collection.training.type qa'
   }
 };
 
+/* ------------ data -------------- */
 export enum DatasetDataIndexTypeEnum {
   chunk = 'chunk',
   qa = 'qa',
@@ -61,29 +101,47 @@ export const DatasetDataIndexTypeMap = {
   }
 };
 
+/* ------------ training -------------- */
 export enum TrainingModeEnum {
-  'chunk' = 'chunk',
-  'qa' = 'qa'
-  // 'hypothetical' = 'hypothetical',
-  // 'summary' = 'summary',
-  // 'multipleIndex' = 'multipleIndex'
+  chunk = 'chunk',
+  qa = 'qa'
 }
+
 export const TrainingTypeMap = {
   [TrainingModeEnum.chunk]: {
-    name: 'chunk'
+    label: 'core.dataset.training.type chunk'
   },
   [TrainingModeEnum.qa]: {
-    name: 'qa'
+    label: 'core.dataset.training.type qa'
+  }
+};
+
+/* ------------ search -------------- */
+export enum DatasetSearchModeEnum {
+  embedding = 'embedding',
+  embeddingReRank = 'embeddingReRank',
+  embFullTextReRank = 'embFullTextReRank'
+}
+
+export const DatasetSearchModeMap = {
+  [DatasetSearchModeEnum.embedding]: {
+    icon: 'core/dataset/modeEmbedding',
+    title: 'core.dataset.search.mode.embedding',
+    desc: 'core.dataset.search.mode.embedding desc',
+    value: DatasetSearchModeEnum.embedding
+  },
+  [DatasetSearchModeEnum.embeddingReRank]: {
+    icon: 'core/dataset/modeEmbeddingRerank',
+    title: 'core.dataset.search.mode.embeddingReRank',
+    desc: 'core.dataset.search.mode.embeddingReRank desc',
+    value: DatasetSearchModeEnum.embeddingReRank
+  },
+  [DatasetSearchModeEnum.embFullTextReRank]: {
+    icon: 'core/dataset/modeEmbFTRerank',
+    title: 'core.dataset.search.mode.embFullTextReRank',
+    desc: 'core.dataset.search.mode.embFullTextReRank desc',
+    value: DatasetSearchModeEnum.embFullTextReRank
   }
-  // [TrainingModeEnum.hypothetical]: {
-  //   name: 'hypothetical'
-  // },
-  // [TrainingModeEnum.summary]: {
-  //   name: 'summary'
-  // },
-  // [TrainingModeEnum.multipleIndex]: {
-  //   name: 'multipleIndex'
-  // }
 };
 
 export const FolderAvatarSrc = '/imgs/files/folder.svg';

packages/global/core/dataset/controller.d.ts

@@ -5,6 +5,7 @@ export type CreateDatasetDataProps = {
   tmbId: string;
   datasetId: string;
   collectionId: string;
+  chunkIndex?: number;
   q: string;
   a?: string;
   indexes?: Omit<DatasetDataIndexItemType, 'dataId'>[];

packages/global/core/dataset/type.d.ts

@@ -4,6 +4,7 @@ import { PushDatasetDataChunkProps } from './api';
 import {
   DatasetCollectionTypeEnum,
   DatasetDataIndexTypeEnum,
+  DatasetStatusEnum,
   DatasetTypeEnum,
   TrainingModeEnum
 } from './constant';
@@ -20,26 +21,31 @@ export type DatasetSchemaType = {
   name: string;
   vectorModel: string;
   agentModel: string;
-  tags: string[];
+  intro: string;
   type: `${DatasetTypeEnum}`;
+  status: `${DatasetStatusEnum}`;
   permission: `${PermissionTypeEnum}`;
+  websiteConfig?: {
+    url: string;
+    selector: string;
+  };
 };
 
 export type DatasetCollectionSchemaType = {
   _id: string;
-  userId: string;
   teamId: string;
   tmbId: string;
   datasetId: string;
   parentId?: string;
   name: string;
   type: `${DatasetCollectionTypeEnum}`;
+  createTime: Date;
   updateTime: Date;
-  metadata: {
-    fileId?: string;
-    rawLink?: string;
-    pgCollectionId?: string;
-  };
+  trainingType: `${TrainingModeEnum}`;
+  chunkSize: number;
+  fileId?: string;
+  rawLink?: string;
+  metadata?: Record<string, any>;
 };
 
 export type DatasetDataIndexItemType = {
@@ -57,8 +63,11 @@ export type DatasetDataSchemaType = {
   collectionId: string;
   datasetId: string;
   collectionId: string;
+  chunkIndex: number;
+  updateTime: Date;
   q: string; // large chunks or question
   a: string; // answer or custom content
+  fullTextToken: string;
   indexes: DatasetDataIndexItemType[];
 };
 
@@ -77,14 +86,30 @@ export type DatasetTrainingSchemaType = {
   prompt: string;
   q: string;
   a: string;
+  chunkIndex: number;
   indexes: Omit<DatasetDataIndexItemType, 'dataId'>[];
 };
 
 export type CollectionWithDatasetType = Omit<DatasetCollectionSchemaType, 'datasetId'> & {
   datasetId: DatasetSchemaType;
 };
+export type DatasetDataWithCollectionType = Omit<DatasetDataSchemaType, 'collectionId'> & {
+  collectionId: DatasetCollectionSchemaType;
+};
 
 /* ================= dataset ===================== */
+export type DatasetListItemType = {
+  _id: string;
+  parentId: string;
+  avatar: string;
+  name: string;
+  intro: string;
+  type: `${DatasetTypeEnum}`;
+  isOwner: boolean;
+  canWrite: boolean;
+  permission: `${PermissionTypeEnum}`;
+  vectorModel: VectorModelItemType;
+};
 export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel'> & {
   vectorModel: VectorModelItemType;
   agentModel: LLMModelItemType;
@@ -97,6 +122,7 @@ export type DatasetCollectionItemType = CollectionWithDatasetType & {
   canWrite: boolean;
   sourceName: string;
   sourceId?: string;
+  file?: DatasetFileSchema;
 };
 
 /* ================= data ===================== */
@@ -130,6 +156,6 @@ export type DatasetFileSchema = {
 };
 
 /* ============= search =============== */
-export type SearchDataResponseItemType = DatasetDataItemType & {
+export type SearchDataResponseItemType = Omit<DatasetDataItemType, 'isOwner' | 'canWrite'> & {
   score: number;
 };

packages/global/core/module/constants.ts

@@ -0,0 +1,104 @@
+export enum ModuleTemplateTypeEnum {
+  userGuide = 'userGuide',
+  systemInput = 'systemInput',
+  textAnswer = 'textAnswer',
+  dataset = 'dataset',
+  functionCall = 'functionCall',
+  externalCall = 'externalCall',
+
+  personalPlugin = 'personalPlugin',
+  communityPlugin = 'communityPlugin',
+  commercialPlugin = 'commercialPlugin',
+
+  other = 'other'
+}
+
+export enum ModuleDataTypeEnum {
+  string = 'string',
+  number = 'number',
+  boolean = 'boolean',
+  chatHistory = 'chatHistory',
+  datasetQuote = 'datasetQuote',
+  any = 'any',
+
+  // plugin special type
+  selectApp = 'selectApp',
+  selectDataset = 'selectDataset'
+}
+
+/* reg: modulename key */
+export enum ModuleInputKeyEnum {
+  // old
+  welcomeText = 'welcomeText',
+  variables = 'variables',
+  switch = 'switch', // a trigger switch
+  history = 'history',
+  userChatInput = 'userChatInput',
+  questionGuide = 'questionGuide',
+  tts = 'tts',
+  answerText = 'text',
+  agents = 'agents', // cq agent key
+
+  // latest
+  // common
+  aiModel = 'model',
+  aiSystemPrompt = 'systemPrompt',
+  description = 'description',
+
+  // history
+  historyMaxAmount = 'maxContext',
+
+  // ai chat
+  aiChatTemperature = 'temperature',
+  aiChatMaxToken = 'maxToken',
+  aiChatSettingModal = 'aiSettings',
+  aiChatIsResponseText = 'isResponseAnswerText',
+  aiChatQuoteTemplate = 'quoteTemplate',
+  aiChatQuotePrompt = 'quotePrompt',
+  aiChatDatasetQuote = 'quoteQA',
+
+  // dataset
+  datasetSelectList = 'datasets',
+  datasetSimilarity = 'similarity',
+  datasetLimit = 'limit',
+  datasetSearchMode = 'searchMode',
+  datasetParamsModal = 'datasetParamsModal',
+
+  // context extract
+  contextExtractInput = 'content',
+  extractKeys = 'extractKeys',
+
+  // http
+  httpUrl = 'url',
+
+  // app
+  runAppSelectApp = 'app',
+
+  // plugin
+  pluginId = 'pluginId'
+}
+
+export enum ModuleOutputKeyEnum {
+  // common
+  userChatInput = 'userChatInput',
+  finish = 'finish',
+  responseData = 'responseData',
+  history = 'history',
+  answerText = 'answerText', //  answer module text key
+  success = 'success',
+  failed = 'failed',
+
+  // dataset
+  datasetIsEmpty = 'isEmpty',
+  datasetUnEmpty = 'unEmpty',
+  datasetQuoteQA = 'quoteQA',
+
+  // context extract
+  contextExtractFields = 'fields'
+}
+
+export enum VariableInputEnum {
+  input = 'input',
+  textarea = 'textarea',
+  select = 'select'
+}

packages/global/core/module/node/constant.ts

@@ -1,5 +1,6 @@
 export enum FlowNodeInputTypeEnum {
   systemInput = 'systemInput', // history, userChatInput, variableInput
+
   input = 'input', // one line input
   textarea = 'textarea',
   numberInput = 'numberInput',
@@ -8,14 +9,15 @@ export enum FlowNodeInputTypeEnum {
   custom = 'custom',
   target = 'target', // data input
   switch = 'switch',
-  chatInput = 'chatInput',
   selectApp = 'selectApp',
   // chat special input
   aiSettings = 'aiSettings',
-  maxToken = 'maxToken',
+  // maxToken = 'maxToken',
   selectChatModel = 'selectChatModel',
   // dataset special input
   selectDataset = 'selectDataset',
+  selectDatasetParamsModal = 'selectDatasetParamsModal',
+
   hidden = 'hidden'
 }
 
@@ -44,18 +46,3 @@ export enum FlowNodeTypeEnum {
   // abandon
   variable = 'variable'
 }
-
-export enum FlowNodeSpecialInputKeyEnum {
-  'answerText' = 'text',
-  'agents' = 'agents', // cq agent key
-  'pluginId' = 'pluginId'
-}
-
-export enum FlowNodeValTypeEnum {
-  'string' = 'string',
-  'number' = 'number',
-  'boolean' = 'boolean',
-  'chatHistory' = 'chatHistory',
-  'datasetQuote' = 'datasetQuote',
-  'any' = 'any'
-}

packages/global/core/module/node/type.d.ts

@@ -1,9 +1,6 @@
-import {
-  FlowNodeInputTypeEnum,
-  FlowNodeOutputTypeEnum,
-  FlowNodeValTypeEnum,
-  FlowNodeTypeEnum
-} from './constant';
+import { FlowNodeInputTypeEnum, FlowNodeOutputTypeEnum, FlowNodeTypeEnum } from './constant';
+import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleOutputKeyEnum } from '../constants';
+import { SelectedDatasetType } from '../api';
 
 export type FlowNodeChangeProps = {
   moduleId: string;
@@ -23,24 +20,27 @@ export type FlowNodeChangeProps = {
 };
 
 export type FlowNodeInputItemType = {
-  key: string; // 字段名
+  key: `${ModuleInputKeyEnum}` | string;
+  type: `${FlowNodeInputTypeEnum}`; // Decide on a render style
   value?: any;
-  valueType?: `${FlowNodeValTypeEnum}`;
-  type: `${FlowNodeInputTypeEnum}`;
+  valueType?: `${ModuleDataTypeEnum}`; // data type
   label: string;
-  edit?: boolean;
-  connected?: boolean;
   description?: string;
-  placeholder?: string;
-  plusField?: boolean;
+  required?: boolean;
+  edit?: boolean; // Whether to allow editing
+  connected?: boolean; // unConnected field will be deleted
+
+  showTargetInApp?: boolean;
+  showTargetInPlugin?: boolean;
+
+  placeholder?: string; // input,textarea
+  list?: { label: string; value: any }[]; // select
+  step?: number; // slider max?: number;
   max?: number;
   min?: number;
-  step?: number;
-  required?: boolean;
-  list?: { label: string; value: any }[];
-  markList?: { label: string; value: any }[];
-  customData?: () => any;
-  valueCheck?: (value: any) => boolean;
+  markList?: { label: string; value: any }[]; // slider
+
+  plusField?: boolean; // plus system will show
 };
 
 export type FlowNodeOutputTargetItemType = {
@@ -48,11 +48,30 @@ export type FlowNodeOutputTargetItemType = {
   key: string;
 };
 export type FlowNodeOutputItemType = {
-  key: string; // 字段名
+  key: `${ModuleOutputKeyEnum}` | string;
   label?: string;
   edit?: boolean;
   description?: string;
-  valueType?: `${FlowNodeValTypeEnum}`;
+  valueType?: `${ModuleDataTypeEnum}`;
   type?: `${FlowNodeOutputTypeEnum}`;
   targets: FlowNodeOutputTargetItemType[];
 };
+
+/* ------------- item type --------------- */
+/* ai chat modules props */
+export type AIChatModuleProps = {
+  [ModuleInputKeyEnum.aiModel]: string;
+  [ModuleInputKeyEnum.aiSystemPrompt]?: string;
+  [ModuleInputKeyEnum.aiChatTemperature]: number;
+  [ModuleInputKeyEnum.aiChatMaxToken]: number;
+  [ModuleInputKeyEnum.aiChatIsResponseText]: boolean;
+  [ModuleInputKeyEnum.aiChatQuoteTemplate]?: string;
+  [ModuleInputKeyEnum.aiChatQuotePrompt]?: string;
+};
+
+export type DatasetModuleProps = {
+  [ModuleInputKeyEnum.datasetSelectList]: SelectedDatasetType;
+  [ModuleInputKeyEnum.datasetSimilarity]: number;
+  [ModuleInputKeyEnum.datasetLimit]: number;
+  [ModuleInputKeyEnum.datasetStartReRank]: boolean;
+};

packages/global/core/module/template/input.ts

@@ -0,0 +1,32 @@
+import type { FlowNodeInputItemType } from '../node/type.d';
+import { ModuleInputKeyEnum } from '../constants';
+import { FlowNodeInputTypeEnum } from '../node/constant';
+import { ModuleDataTypeEnum } from '../constants';
+
+export const Input_Template_TFSwitch: FlowNodeInputItemType = {
+  key: ModuleInputKeyEnum.switch,
+  type: FlowNodeInputTypeEnum.target,
+  label: 'core.module.input.label.switch',
+  valueType: ModuleDataTypeEnum.any,
+  showTargetInApp: true,
+  showTargetInPlugin: true
+};
+
+export const Input_Template_History: FlowNodeInputItemType = {
+  key: ModuleInputKeyEnum.history,
+  type: FlowNodeInputTypeEnum.target,
+  label: 'core.module.input.label.chat history',
+  valueType: ModuleDataTypeEnum.chatHistory,
+  showTargetInApp: true,
+  showTargetInPlugin: true
+};
+
+export const Input_Template_UserChatInput: FlowNodeInputItemType = {
+  key: ModuleInputKeyEnum.userChatInput,
+  type: FlowNodeInputTypeEnum.target,
+  label: 'core.module.input.label.user question',
+  required: true,
+  valueType: ModuleDataTypeEnum.string,
+  showTargetInApp: true,
+  showTargetInPlugin: true
+};

packages/global/core/module/template/output.ts

@@ -0,0 +1,13 @@
+import type { FlowNodeOutputItemType } from '../node/type';
+import { ModuleOutputKeyEnum } from '../constants';
+import { FlowNodeOutputTypeEnum } from '../node/constant';
+import { ModuleDataTypeEnum } from '../constants';
+
+export const Output_Template_Finish: FlowNodeOutputItemType = {
+  key: ModuleOutputKeyEnum.finish,
+  label: 'core.module.output.label.running done',
+  description: 'core.module.output.description.running done',
+  valueType: ModuleDataTypeEnum.boolean,
+  type: FlowNodeOutputTypeEnum.source,
+  targets: []
+};

packages/global/core/module/template/system/aiChat.ts

@@ -0,0 +1,157 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import {
+  ModuleDataTypeEnum,
+  ModuleInputKeyEnum,
+  ModuleOutputKeyEnum,
+  ModuleTemplateTypeEnum
+} from '../../constants';
+import {
+  Input_Template_History,
+  Input_Template_TFSwitch,
+  Input_Template_UserChatInput
+} from '../input';
+import { chatNodeSystemPromptTip } from '../tip';
+import { Output_Template_Finish } from '../output';
+
+export const AiChatModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.chatNode,
+  templateType: ModuleTemplateTypeEnum.textAnswer,
+  flowType: FlowNodeTypeEnum.chatNode,
+  avatar: '/imgs/module/AI.png',
+  name: 'AI 对话',
+  intro: 'AI 大模型对话',
+  showStatus: true,
+  inputs: [
+    Input_Template_TFSwitch,
+    {
+      key: ModuleInputKeyEnum.aiModel,
+      type: FlowNodeInputTypeEnum.selectChatModel,
+      label: '对话模型',
+      required: true,
+      valueType: ModuleDataTypeEnum.string,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    // --- settings modal
+    {
+      key: ModuleInputKeyEnum.aiChatTemperature,
+      type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window
+      label: '温度',
+      value: 0,
+      valueType: ModuleDataTypeEnum.number,
+      min: 0,
+      max: 10,
+      step: 1,
+      markList: [
+        { label: '严谨', value: 0 },
+        { label: '发散', value: 10 }
+      ],
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.aiChatMaxToken,
+      type: FlowNodeInputTypeEnum.hidden, // Set in the pop-up window
+      label: '回复上限',
+      value: 2000,
+      valueType: ModuleDataTypeEnum.number,
+      min: 100,
+      max: 4000,
+      step: 50,
+      markList: [
+        { label: '100', value: 100 },
+        {
+          label: `${4000}`,
+          value: 4000
+        }
+      ],
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.aiChatIsResponseText,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: '返回AI内容',
+      value: true,
+      valueType: ModuleDataTypeEnum.boolean,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.aiChatQuoteTemplate,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: '引用内容模板',
+      valueType: ModuleDataTypeEnum.string,
+      value: '',
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.aiChatQuotePrompt,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: '引用内容提示词',
+      valueType: ModuleDataTypeEnum.string,
+      value: '',
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.aiChatSettingModal,
+      type: FlowNodeInputTypeEnum.aiSettings,
+      label: '',
+      connected: false,
+      valueType: ModuleDataTypeEnum.any,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    // settings modal ---
+    {
+      key: ModuleInputKeyEnum.aiSystemPrompt,
+      type: FlowNodeInputTypeEnum.textarea,
+      label: '系统提示词',
+      max: 300,
+      valueType: ModuleDataTypeEnum.string,
+      description: chatNodeSystemPromptTip,
+      placeholder: chatNodeSystemPromptTip,
+      value: '',
+      showTargetInApp: true,
+      showTargetInPlugin: true
+    },
+    {
+      key: ModuleInputKeyEnum.aiChatDatasetQuote,
+      type: FlowNodeInputTypeEnum.target,
+      label: '引用内容',
+      description: "对象数组格式，结构：\n [{q:'问题',a:'回答'}]",
+      valueType: ModuleDataTypeEnum.datasetQuote,
+      connected: false,
+      showTargetInApp: true,
+      showTargetInPlugin: true
+    },
+    Input_Template_History,
+    Input_Template_UserChatInput
+  ],
+  outputs: [
+    {
+      key: ModuleOutputKeyEnum.history,
+      label: '新的上下文',
+      description: '将本次回复内容拼接上历史记录，作为新的上下文返回',
+      valueType: ModuleDataTypeEnum.chatHistory,
+      type: FlowNodeOutputTypeEnum.source,
+      targets: []
+    },
+    {
+      key: ModuleOutputKeyEnum.answerText,
+      label: 'AI回复',
+      description: '将在 stream 回复完毕后触发',
+      valueType: ModuleDataTypeEnum.string,
+      type: FlowNodeOutputTypeEnum.source,
+      targets: []
+    },
+    Output_Template_Finish
+  ]
+};

packages/global/core/module/template/system/assignedAnswer.ts

@@ -0,0 +1,29 @@
+import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+import { Input_Template_TFSwitch } from '../input';
+import { Output_Template_Finish } from '../output';
+
+export const AssignedAnswerModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.answerNode,
+  templateType: ModuleTemplateTypeEnum.textAnswer,
+  flowType: FlowNodeTypeEnum.answerNode,
+  avatar: '/imgs/module/reply.png',
+  name: '指定回复',
+  intro: '该模块可以直接回复一段指定的内容。常用于引导、提示',
+  inputs: [
+    Input_Template_TFSwitch,
+    {
+      key: ModuleInputKeyEnum.answerText,
+      type: FlowNodeInputTypeEnum.textarea,
+      valueType: ModuleDataTypeEnum.any,
+      value: '',
+      label: '回复的内容',
+      description:
+        '可以使用 \\n 来实现连续换行。\n\n可以通过外部模块输入实现回复，外部模块输入时会覆盖当前填写的内容。\n\n如传入非字符串类型数据将会自动转成字符串',
+      showTargetInApp: true,
+      showTargetInPlugin: true
+    }
+  ],
+  outputs: [Output_Template_Finish]
+};

packages/global/core/module/template/system/classifyQuestion.ts

@@ -0,0 +1,96 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+import {
+  Input_Template_History,
+  Input_Template_TFSwitch,
+  Input_Template_UserChatInput
+} from '../input';
+
+export const ClassifyQuestionModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.classifyQuestion,
+  templateType: ModuleTemplateTypeEnum.functionCall,
+  flowType: FlowNodeTypeEnum.classifyQuestion,
+  avatar: '/imgs/module/cq.png',
+  name: '问题分类',
+  intro: `根据用户的历史记录和当前问题判断该次提问的类型。可以添加多组问题类型，下面是一个模板例子：
+类型1: 打招呼
+类型2: 关于商品“使用”问题
+类型3: 关于商品“购买”问题
+类型4: 其他问题`,
+  showStatus: true,
+  inputs: [
+    Input_Template_TFSwitch,
+    {
+      key: ModuleInputKeyEnum.aiModel,
+      type: FlowNodeInputTypeEnum.selectChatModel,
+      valueType: ModuleDataTypeEnum.string,
+      label: '分类模型',
+      required: true,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.aiSystemPrompt,
+      type: FlowNodeInputTypeEnum.textarea,
+      valueType: ModuleDataTypeEnum.string,
+      value: '',
+      label: '背景知识',
+      description:
+        '你可以添加一些特定内容的介绍，从而更好的识别用户的问题类型。这个内容通常是给模型介绍一个它不知道的内容。',
+      placeholder:
+        '例如: \n1. AIGC（人工智能生成内容）是指使用人工智能技术自动或半自动地生成数字内容，如文本、图像、音乐、视频等。\n2. AIGC技术包括但不限于自然语言处理、计算机视觉、机器学习和深度学习。这些技术可以创建新内容或修改现有内容，以满足特定的创意、教育、娱乐或信息需求。',
+      showTargetInApp: true,
+      showTargetInPlugin: true
+    },
+    Input_Template_History,
+    Input_Template_UserChatInput,
+    {
+      key: ModuleInputKeyEnum.agents,
+      type: FlowNodeInputTypeEnum.custom,
+      valueType: ModuleDataTypeEnum.any,
+      label: '',
+      value: [
+        {
+          value: '打招呼',
+          key: 'fasw'
+        },
+        {
+          value: '关于 xxx 的问题',
+          key: 'fqsw'
+        },
+        {
+          value: '其他问题',
+          key: 'fesw'
+        }
+      ],
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    }
+  ],
+  outputs: [
+    // custom output
+    {
+      key: 'fasw',
+      label: '',
+      type: FlowNodeOutputTypeEnum.hidden,
+      targets: []
+    },
+    {
+      key: 'fqsw',
+      label: '',
+      type: FlowNodeOutputTypeEnum.hidden,
+      targets: []
+    },
+    {
+      key: 'fesw',
+      label: '',
+      type: FlowNodeOutputTypeEnum.hidden,
+      targets: []
+    }
+  ]
+};

packages/global/core/module/template/system/contextExtract.ts

@@ -0,0 +1,83 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import {
+  ModuleDataTypeEnum,
+  ModuleInputKeyEnum,
+  ModuleOutputKeyEnum,
+  ModuleTemplateTypeEnum
+} from '../../constants';
+import { Input_Template_History, Input_Template_TFSwitch } from '../input';
+
+export const ContextExtractModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.contentExtract,
+  templateType: ModuleTemplateTypeEnum.functionCall,
+  flowType: FlowNodeTypeEnum.contentExtract,
+  avatar: '/imgs/module/extract.png',
+  name: '文本内容提取',
+  intro: '可从文本中提取指定的数据，例如：sql语句、搜索关键词、代码等',
+  showStatus: true,
+  inputs: [
+    Input_Template_TFSwitch,
+    {
+      key: ModuleInputKeyEnum.description,
+      type: FlowNodeInputTypeEnum.textarea,
+      valueType: ModuleDataTypeEnum.string,
+      value: '',
+      label: '提取要求描述',
+      description: '写一段提取要求，告诉 AI 需要提取哪些内容',
+      required: true,
+      placeholder:
+        '例如: \n1. 你是一个实验室预约助手。根据用户问题，提取出姓名、实验室号和预约时间',
+      showTargetInApp: true,
+      showTargetInPlugin: true
+    },
+    Input_Template_History,
+    {
+      key: ModuleInputKeyEnum.contextExtractInput,
+      type: FlowNodeInputTypeEnum.target,
+      label: '需要提取的文本',
+      required: true,
+      valueType: ModuleDataTypeEnum.string,
+      showTargetInApp: true,
+      showTargetInPlugin: true
+    },
+    {
+      key: ModuleInputKeyEnum.extractKeys,
+      type: FlowNodeInputTypeEnum.custom,
+      label: '目标字段',
+      valueType: ModuleDataTypeEnum.any,
+      description: "由 '描述' 和 'key' 组成一个目标字段，可提取多个目标字段",
+      value: [], // {desc: string; key: string; required: boolean;}[]
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    }
+  ],
+  outputs: [
+    {
+      key: ModuleOutputKeyEnum.success,
+      label: '字段完全提取',
+      valueType: ModuleDataTypeEnum.boolean,
+      type: FlowNodeOutputTypeEnum.source,
+      targets: []
+    },
+    {
+      key: ModuleOutputKeyEnum.failed,
+      label: '提取字段缺失',
+      valueType: ModuleDataTypeEnum.boolean,
+      type: FlowNodeOutputTypeEnum.source,
+      targets: []
+    },
+    {
+      key: ModuleOutputKeyEnum.contextExtractFields,
+      label: '完整提取结果',
+      description: '一个 JSON 字符串，例如：{"name:":"YY","Time":"2023/7/2 18:00"}',
+      valueType: ModuleDataTypeEnum.string,
+      type: FlowNodeOutputTypeEnum.source,
+      targets: []
+    }
+  ]
+};

packages/global/core/module/template/system/datasetSearch.ts

@@ -0,0 +1,117 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import {
+  ModuleDataTypeEnum,
+  ModuleInputKeyEnum,
+  ModuleOutputKeyEnum,
+  ModuleTemplateTypeEnum
+} from '../../constants';
+import { Input_Template_TFSwitch, Input_Template_UserChatInput } from '../input';
+import { Output_Template_Finish } from '../output';
+import { DatasetSearchModeEnum } from '../../../dataset/constant';
+
+export const DatasetSearchModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.datasetSearchNode,
+  templateType: ModuleTemplateTypeEnum.dataset,
+  flowType: FlowNodeTypeEnum.datasetSearchNode,
+  avatar: '/imgs/module/db.png',
+  name: '知识库搜索',
+  intro: '去知识库中搜索对应的答案。可作为 AI 对话引用参考。',
+  showStatus: true,
+  inputs: [
+    Input_Template_TFSwitch,
+    {
+      key: ModuleInputKeyEnum.datasetSelectList,
+      type: FlowNodeInputTypeEnum.selectDataset,
+      label: '关联的知识库',
+      value: [],
+      valueType: ModuleDataTypeEnum.selectDataset,
+      list: [],
+      required: true,
+      showTargetInApp: false,
+      showTargetInPlugin: true
+    },
+    {
+      key: ModuleInputKeyEnum.datasetSimilarity,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: '最低相关性',
+      value: 0.4,
+      valueType: ModuleDataTypeEnum.number,
+      min: 0,
+      max: 1,
+      step: 0.01,
+      markList: [
+        { label: '0', value: 0 },
+        { label: '1', value: 1 }
+      ],
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.datasetLimit,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: '单次搜索上限',
+      description: '最多取 n 条记录作为本次问题引用',
+      value: 5,
+      valueType: ModuleDataTypeEnum.number,
+      min: 1,
+      max: 20,
+      step: 1,
+      markList: [
+        { label: '1', value: 1 },
+        { label: '20', value: 20 }
+      ],
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.datasetSearchMode,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: 'core.dataset.search.Mode',
+      valueType: ModuleDataTypeEnum.string,
+      showTargetInApp: false,
+      showTargetInPlugin: false,
+      value: DatasetSearchModeEnum.embedding
+    },
+    {
+      key: ModuleInputKeyEnum.datasetParamsModal,
+      type: FlowNodeInputTypeEnum.selectDatasetParamsModal,
+      label: '',
+      connected: false,
+      valueType: ModuleDataTypeEnum.any,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    Input_Template_UserChatInput
+  ],
+  outputs: [
+    {
+      key: ModuleOutputKeyEnum.datasetIsEmpty,
+      label: '搜索结果为空',
+      type: FlowNodeOutputTypeEnum.source,
+      valueType: ModuleDataTypeEnum.boolean,
+      targets: []
+    },
+    {
+      key: ModuleOutputKeyEnum.datasetUnEmpty,
+      label: '搜索结果不为空',
+      type: FlowNodeOutputTypeEnum.source,
+      valueType: ModuleDataTypeEnum.boolean,
+      targets: []
+    },
+    {
+      key: ModuleOutputKeyEnum.datasetQuoteQA,
+      label: '引用内容',
+      description:
+        '始终返回数组，如果希望搜索结果为空时执行额外操作，需要用到上面的两个输入以及目标模块的触发器',
+      type: FlowNodeOutputTypeEnum.source,
+      valueType: ModuleDataTypeEnum.datasetQuote,
+      targets: []
+    },
+    Output_Template_Finish
+  ]
+};

packages/global/core/module/template/system/empty.ts

@@ -0,0 +1,14 @@
+import { ModuleTemplateTypeEnum } from '../../constants';
+import { FlowNodeTypeEnum } from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+
+export const EmptyModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.empty,
+  templateType: ModuleTemplateTypeEnum.other,
+  flowType: FlowNodeTypeEnum.empty,
+  avatar: '/imgs/module/cq.png',
+  name: '该模块已被移除',
+  intro: '',
+  inputs: [],
+  outputs: []
+};

packages/global/core/module/template/system/history.ts

@@ -0,0 +1,48 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+
+export const HistoryModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.historyNode,
+  templateType: ModuleTemplateTypeEnum.systemInput,
+  flowType: FlowNodeTypeEnum.historyNode,
+  avatar: '/imgs/module/history.png',
+  name: '聊天记录',
+  intro: '用户输入的内容。该模块通常作为应用的入口，用户在发送消息后会首先执行该模块。',
+  inputs: [
+    {
+      key: ModuleInputKeyEnum.historyMaxAmount,
+      type: FlowNodeInputTypeEnum.numberInput,
+      label: '最长记录数',
+      description:
+        '该记录数不代表模型可接收这么多的历史记录，具体可接收多少历史记录，取决于模型的能力，通常建议不要超过20条。',
+      value: 6,
+      valueType: ModuleDataTypeEnum.number,
+      min: 0,
+      max: 100,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.history,
+      type: FlowNodeInputTypeEnum.hidden,
+      valueType: ModuleDataTypeEnum.chatHistory,
+      label: '聊天记录',
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    }
+  ],
+  outputs: [
+    {
+      key: ModuleInputKeyEnum.history,
+      label: '聊天记录',
+      valueType: ModuleDataTypeEnum.chatHistory,
+      type: FlowNodeOutputTypeEnum.source,
+      targets: []
+    }
+  ]
+};

packages/global/core/module/template/system/http.ts

@@ -0,0 +1,31 @@
+import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+import { Input_Template_TFSwitch } from '../input';
+import { Output_Template_Finish } from '../output';
+
+export const HttpModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.httpRequest,
+  templateType: ModuleTemplateTypeEnum.externalCall,
+  flowType: FlowNodeTypeEnum.httpRequest,
+  avatar: '/imgs/module/http.png',
+  name: 'HTTP模块',
+  intro: '可以发出一个 HTTP POST 请求，实现更为复杂的操作（联网搜索、数据库查询等）',
+  showStatus: true,
+  inputs: [
+    Input_Template_TFSwitch,
+    {
+      key: ModuleInputKeyEnum.httpUrl,
+      value: '',
+      type: FlowNodeInputTypeEnum.input,
+      valueType: ModuleDataTypeEnum.string,
+      label: '请求地址',
+      description: '请求目标地址',
+      placeholder: 'https://api.fastgpt.run/getInventory',
+      required: true,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    }
+  ],
+  outputs: [Output_Template_Finish]
+};

packages/global/core/module/template/system/pluginInput.ts

@@ -0,0 +1,15 @@
+import { ModuleTemplateTypeEnum } from '../../constants';
+import { FlowNodeTypeEnum } from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+
+export const PluginInputModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.pluginInput,
+  templateType: ModuleTemplateTypeEnum.systemInput,
+  flowType: FlowNodeTypeEnum.pluginInput,
+  avatar: '/imgs/module/input.png',
+  name: '定义插件输入',
+  intro: '自定义配置外部输入，使用插件时，仅暴露自定义配置的输入',
+  showStatus: false,
+  inputs: [],
+  outputs: []
+};

packages/global/core/module/template/system/pluginOutput.ts

@@ -0,0 +1,15 @@
+import { ModuleTemplateTypeEnum } from '../../constants';
+import { FlowNodeTypeEnum } from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+
+export const PluginOutputModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.pluginOutput,
+  templateType: ModuleTemplateTypeEnum.systemInput,
+  flowType: FlowNodeTypeEnum.pluginOutput,
+  avatar: '/imgs/module/output.png',
+  name: '定义插件输出',
+  intro: '自定义配置外部输出，使用插件时，仅暴露自定义配置的输出',
+  showStatus: false,
+  inputs: [],
+  outputs: []
+};

packages/global/core/module/template/system/runApp.ts

@@ -0,0 +1,62 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import {
+  ModuleDataTypeEnum,
+  ModuleInputKeyEnum,
+  ModuleOutputKeyEnum,
+  ModuleTemplateTypeEnum
+} from '../../constants';
+import {
+  Input_Template_History,
+  Input_Template_TFSwitch,
+  Input_Template_UserChatInput
+} from '../input';
+import { Output_Template_Finish } from '../output';
+
+export const RunAppModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.runApp,
+  templateType: ModuleTemplateTypeEnum.externalCall,
+  flowType: FlowNodeTypeEnum.runApp,
+  avatar: '/imgs/module/app.png',
+  name: '应用调用',
+  intro: '可以选择一个其他应用进行调用',
+  showStatus: true,
+  inputs: [
+    Input_Template_TFSwitch,
+    {
+      key: ModuleInputKeyEnum.runAppSelectApp,
+      type: FlowNodeInputTypeEnum.selectApp,
+      valueType: ModuleDataTypeEnum.selectApp,
+      label: '选择一个应用',
+      description: '选择一个其他应用进行调用',
+      required: true,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    Input_Template_History,
+    Input_Template_UserChatInput
+  ],
+  outputs: [
+    {
+      key: ModuleOutputKeyEnum.history,
+      label: '新的上下文',
+      description: '将该应用回复内容拼接到历史记录中，作为新的上下文返回',
+      valueType: ModuleDataTypeEnum.chatHistory,
+      type: FlowNodeOutputTypeEnum.source,
+      targets: []
+    },
+    {
+      key: ModuleOutputKeyEnum.answerText,
+      label: 'AI回复',
+      description: '将在应用完全结束后触发',
+      valueType: ModuleDataTypeEnum.string,
+      type: FlowNodeOutputTypeEnum.source,
+      targets: []
+    },
+    Output_Template_Finish
+  ]
+};

packages/global/core/module/template/system/runPlugin.ts

@@ -0,0 +1,15 @@
+import { ModuleTemplateTypeEnum } from '../../constants';
+import { FlowNodeTypeEnum } from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+
+export const RunPluginModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.pluginModule,
+  templateType: ModuleTemplateTypeEnum.externalCall,
+  flowType: FlowNodeTypeEnum.pluginModule,
+  avatar: '/imgs/module/custom.png',
+  intro: '',
+  name: '自定义模块',
+  showStatus: false,
+  inputs: [], // [{key:'pluginId'},...]
+  outputs: []
+};

packages/global/core/module/template/system/userGuide.ts

@@ -0,0 +1,49 @@
+import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import { userGuideTip } from '../tip';
+import { ModuleDataTypeEnum, ModuleInputKeyEnum, ModuleTemplateTypeEnum } from '../../constants';
+
+export const UserGuideModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.userGuide,
+  templateType: ModuleTemplateTypeEnum.userGuide,
+  flowType: FlowNodeTypeEnum.userGuide,
+  avatar: '/imgs/module/userGuide.png',
+  name: '用户引导',
+  intro: userGuideTip,
+  inputs: [
+    {
+      key: ModuleInputKeyEnum.welcomeText,
+      type: FlowNodeInputTypeEnum.hidden,
+      valueType: ModuleDataTypeEnum.string,
+      label: '开场白',
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.variables,
+      type: FlowNodeInputTypeEnum.hidden,
+      valueType: ModuleDataTypeEnum.any,
+      label: '对话框变量',
+      value: [],
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.questionGuide,
+      valueType: ModuleDataTypeEnum.boolean,
+      type: FlowNodeInputTypeEnum.switch,
+      label: '问题引导',
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    },
+    {
+      key: ModuleInputKeyEnum.tts,
+      type: FlowNodeInputTypeEnum.hidden,
+      valueType: ModuleDataTypeEnum.any,
+      label: '语音播报',
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    }
+  ],
+  outputs: []
+};

packages/global/core/module/template/system/userInput.ts

@@ -0,0 +1,40 @@
+import {
+  FlowNodeInputTypeEnum,
+  FlowNodeOutputTypeEnum,
+  FlowNodeTypeEnum
+} from '../../node/constant';
+import { FlowModuleTemplateType } from '../../type.d';
+import {
+  ModuleDataTypeEnum,
+  ModuleInputKeyEnum,
+  ModuleOutputKeyEnum,
+  ModuleTemplateTypeEnum
+} from '../../constants';
+
+export const UserInputModule: FlowModuleTemplateType = {
+  id: FlowNodeTypeEnum.questionInput,
+  templateType: ModuleTemplateTypeEnum.systemInput,
+  flowType: FlowNodeTypeEnum.questionInput,
+  avatar: '/imgs/module/userChatInput.png',
+  name: '用户问题(入口)',
+  intro: '用户输入的内容。该模块通常作为应用的入口，用户在发送消息后会首先执行该模块。',
+  inputs: [
+    {
+      key: ModuleInputKeyEnum.userChatInput,
+      type: FlowNodeInputTypeEnum.systemInput,
+      valueType: ModuleDataTypeEnum.string,
+      label: '用户问题',
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    }
+  ],
+  outputs: [
+    {
+      key: ModuleOutputKeyEnum.userChatInput,
+      label: '用户问题',
+      type: FlowNodeOutputTypeEnum.source,
+      valueType: ModuleDataTypeEnum.string,
+      targets: []
+    }
+  ]
+};

packages/global/core/module/template/tip.ts

@@ -0,0 +1,7 @@
+export const chatNodeSystemPromptTip =
+  '模型固定的引导词，通过调整该内容，可以引导模型聊天方向。该内容会被固定在上下文的开头。可使用变量，例如 {{language}}';
+export const userGuideTip = '可以在对话前设置引导语，设置全局变量，设置下一步指引';
+export const welcomeTextTip =
+  '每次对话开始前，发送一个初始内容。支持标准 Markdown 语法，可使用的额外标记:\n[快捷按键]: 用户点击后可以直接发送该问题';
+export const variableTip =
+  '可以在对话开始前，要求用户填写一些内容作为本轮对话的特定变量。该模块位于开场引导之后。\n变量可以通过 {{变量key}} 的形式注入到其他模块 string 类型的输入中，例如：提示词、限定词等';

packages/global/core/module/type.d.ts

@@ -1,13 +1,14 @@
-import { FlowNodeTypeEnum, FlowNodeValTypeEnum } from './node/constant';
+import { FlowNodeTypeEnum } from './node/constant';
+import { ModuleDataTypeEnum, ModuleTemplateTypeEnum, VariableInputEnum } from './constants';
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
 
 export type FlowModuleTemplateType = {
   id: string;
+  templateType: `${ModuleTemplateTypeEnum}`;
   flowType: `${FlowNodeTypeEnum}`; // unique
-  logo?: string;
+  avatar?: string;
   name: string;
-  description?: string;
-  intro?: string;
+  intro: string; // template list intro
   showStatus?: boolean; // chatting response step status
   inputs: FlowNodeInputItemType[];
   outputs: FlowNodeOutputItemType[];
@@ -15,16 +16,17 @@ export type FlowModuleTemplateType = {
 export type FlowModuleItemType = FlowModuleTemplateType & {
   moduleId: string;
 };
-export type SystemModuleTemplateType = {
+export type moduleTemplateListType = {
+  type: `${ModuleTemplateTypeEnum}`;
   label: string;
   list: FlowModuleTemplateType[];
 }[];
 
+// store module type
 export type ModuleItemType = {
   name: string;
-  logo?: string;
+  avatar?: string;
   intro?: string;
-  description?: string;
   moduleId: string;
   position?: {
     x: number;
@@ -37,6 +39,24 @@ export type ModuleItemType = {
 };
 
 /* function type */
+// variable
+export type VariableItemType = {
+  id: string;
+  key: string;
+  label: string;
+  type: `${VariableInputEnum}`;
+  required: boolean;
+  maxLen: number;
+  enums: { value: string }[];
+};
+// tts
+export type AppTTSConfigType = {
+  type: 'none' | 'web' | 'model';
+  model?: string;
+  voice?: string;
+  speed?: number;
+};
+
 export type SelectAppItemType = {
   id: string;
   name: string;

packages/global/core/module/utils.ts

@@ -1,28 +1,54 @@
-import {
-  FlowNodeInputTypeEnum,
-  FlowNodeSpecialInputKeyEnum,
-  FlowNodeTypeEnum
-} from './node/constant';
+import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from './node/constant';
+import { ModuleDataTypeEnum, ModuleInputKeyEnum } from './constants';
 import { FlowNodeInputItemType, FlowNodeOutputItemType } from './node/type';
-import { ModuleItemType } from './type';
+import { AppTTSConfigType, ModuleItemType, VariableItemType } from './type';
+
+export const getGuideModule = (modules: ModuleItemType[]) =>
+  modules.find((item) => item.flowType === FlowNodeTypeEnum.userGuide);
+
+export const splitGuideModule = (guideModules?: ModuleItemType) => {
+  const welcomeText: string =
+    guideModules?.inputs?.find((item) => item.key === ModuleInputKeyEnum.welcomeText)?.value || '';
+
+  const variableModules: VariableItemType[] =
+    guideModules?.inputs.find((item) => item.key === ModuleInputKeyEnum.variables)?.value || [];
+
+  const questionGuide: boolean =
+    !!guideModules?.inputs?.find((item) => item.key === ModuleInputKeyEnum.questionGuide)?.value ||
+    false;
+
+  const ttsConfig: AppTTSConfigType = guideModules?.inputs?.find(
+    (item) => item.key === ModuleInputKeyEnum.tts
+  )?.value || { type: 'web' };
 
-export function getPluginTemplatePluginIdInput(pluginId: string) {
   return {
-    key: FlowNodeSpecialInputKeyEnum.pluginId,
-    type: FlowNodeInputTypeEnum.hidden,
-    label: 'pluginId',
-    value: pluginId,
-    connected: true
+    welcomeText,
+    variableModules,
+    questionGuide,
+    ttsConfig
   };
-}
+};
 
-export function formatPluginIOModules(
+export function formatPluginToPreviewModule(
   pluginId: string,
   modules: ModuleItemType[]
 ): {
   inputs: FlowNodeInputItemType[];
   outputs: FlowNodeOutputItemType[];
 } {
+  function getPluginTemplatePluginIdInput(pluginId: string): FlowNodeInputItemType {
+    return {
+      key: ModuleInputKeyEnum.pluginId,
+      type: FlowNodeInputTypeEnum.hidden,
+      label: 'pluginId',
+      value: pluginId,
+      valueType: ModuleDataTypeEnum.string,
+      connected: true,
+      showTargetInApp: false,
+      showTargetInPlugin: false
+    };
+  }
+
   const pluginInput = modules.find((module) => module.flowType === FlowNodeTypeEnum.pluginInput);
   const customOutput = modules.find((module) => module.flowType === FlowNodeTypeEnum.pluginOutput);
 

packages/global/core/plugin/constants.ts

@@ -1,8 +1,9 @@
+import { ModuleTemplateTypeEnum } from '../module/constants';
 import { ModuleItemType } from '../module/type';
 
 export const defaultModules: ModuleItemType[] = [
   {
-    moduleId: 'fph4s3',
+    moduleId: 'custom-output',
     name: '自定义输出',
     flowType: 'pluginOutput',
     showStatus: false,
@@ -14,7 +15,7 @@ export const defaultModules: ModuleItemType[] = [
     outputs: []
   },
   {
-    moduleId: 'w09v30',
+    moduleId: 'custom-input',
     name: '自定义输入',
     flowType: 'pluginInput',
     showStatus: false,
@@ -26,3 +27,14 @@ export const defaultModules: ModuleItemType[] = [
     outputs: []
   }
 ];
+
+export enum PluginTypeEnum {
+  personal = 'personal',
+  community = 'community',
+  commercial = 'commercial'
+}
+export const PluginType2TemplateTypeMap = {
+  [PluginTypeEnum.personal]: ModuleTemplateTypeEnum.personalPlugin,
+  [PluginTypeEnum.community]: ModuleTemplateTypeEnum.communityPlugin,
+  [PluginTypeEnum.commercial]: ModuleTemplateTypeEnum.commercialPlugin
+};

packages/global/core/plugin/type.d.ts

@@ -1,4 +1,6 @@
+import { ModuleTemplateTypeEnum } from 'core/module/constants';
 import type { ModuleItemType } from '../module/type.d';
+import { PluginTypeEnum } from './constants';
 
 export type PluginItemSchema = {
   _id: string;
@@ -11,3 +13,14 @@ export type PluginItemSchema = {
   updateTime: Date;
   modules: ModuleItemType[];
 };
+
+/* plugin template */
+export type PluginTemplateType = {
+  id: string;
+  type: `${PluginTypeEnum}`;
+  name: string;
+  avatar: string;
+  intro: string;
+  modules: ModuleItemType[];
+  templateType?: `${ModuleTemplateTypeEnum}`;
+};

packages/global/package.json

@@ -3,13 +3,16 @@
   "version": "1.0.0",
   "dependencies": {
     "axios": "^1.5.1",
-    "timezones-list": "^3.0.2",
+    "cheerio": "1.0.0-rc.12",
     "dayjs": "^1.11.7",
     "encoding": "^0.1.13",
+    "js-tiktoken": "^1.0.7",
+    "node-html-markdown": "^1.3.0",
     "openai": "^4.16.1",
-    "js-tiktoken": "^1.0.7"
+    "timezones-list": "^3.0.2"
   },
   "devDependencies": {
-    "@types/node": "^20.8.5"
+    "@types/node": "^20.8.5",
+    "@types/turndown": "^5.0.4"
   }
 }

packages/global/support/user/api.d.ts

@@ -13,3 +13,8 @@ export type OauthLoginProps = {
   inviterId?: string;
   tmbId?: string;
 };
+
+export type FastLoginProps = {
+  token: string;
+  code: string;
+};

packages/global/support/wallet/bill/tools.ts

@@ -1,6 +1,7 @@
 /* bill common  */
 import { PRICE_SCALE } from './constants';
-import { BillItemType, BillSchema } from './type';
+import { BillSourceEnum } from './constants';
+import { AuthUserTypeEnum } from '../../permission/constant';
 
 /**
  * dataset price / PRICE_SCALE = real price
@@ -8,3 +9,15 @@ import { BillItemType, BillSchema } from './type';
 export const formatPrice = (val = 0, multiple = 1) => {
   return Number(((val / PRICE_SCALE) * multiple).toFixed(10));
 };
+
+export const getBillSourceByAuthType = ({
+  shareId,
+  authType
+}: {
+  shareId?: string;
+  authType?: `${AuthUserTypeEnum}`;
+}) => {
+  if (shareId) return BillSourceEnum.shareLink;
+  if (authType === AuthUserTypeEnum.apikey) return BillSourceEnum.api;
+  return BillSourceEnum.fastgpt;
+};

packages/service/common/api/plusRequest.ts

@@ -101,18 +101,18 @@ export function request(url: string, data: any, config: ConfigType, method: Meth
  * @param {Object} config
  * @returns
  */
-export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+export function GET<T = undefined>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
   return request(url, params, config, 'GET');
 }
 
-export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function POST<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
   return request(url, data, config, 'POST');
 }
 
-export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function PUT<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
   return request(url, data, config, 'PUT');
 }
 
-export function DELETE<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function DELETE<T = undefined>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
   return request(url, data, config, 'DELETE');
 }

packages/service/common/file/gridfs/controller.ts

@@ -89,7 +89,7 @@ export async function delFileById({
   return true;
 }
 
-export async function getDownloadBuf({
+export async function getDownloadStream({
   bucketName,
   fileId
 }: {
@@ -98,14 +98,5 @@ export async function getDownloadBuf({
 }) {
   const bucket = getGridBucket(bucketName);
 
-  const stream = bucket.openDownloadStream(new Types.ObjectId(fileId));
-
-  const buf: Buffer = await new Promise((resolve, reject) => {
-    const buffers: Buffer[] = [];
-    stream.on('data', (data) => buffers.push(data));
-    stream.on('error', reject);
-    stream.on('end', () => resolve(Buffer.concat(buffers)));
-  });
-
-  return buf;
+  return bucket.openDownloadStream(new Types.ObjectId(fileId));
 }

packages/service/common/mongo/init.ts

@@ -26,9 +26,9 @@ export async function connectMongo({
       bufferCommands: true,
       maxConnecting: Number(process.env.DB_MAX_LINK || 5),
       maxPoolSize: Number(process.env.DB_MAX_LINK || 5),
-      minPoolSize: 2,
-      connectTimeoutMS: 20000,
-      waitQueueTimeoutMS: 20000
+      minPoolSize: Number(process.env.DB_MAX_LINK || 10) * 0.5,
+      connectTimeoutMS: 60000,
+      waitQueueTimeoutMS: 60000
     });
 
     console.log('mongo connected');

packages/service/common/response/index.ts

@@ -102,13 +102,13 @@ export function responseWriteController({
   readStream: any;
 }) {
   res.on('drain', () => {
-    readStream.resume();
+    readStream?.resume?.();
   });
 
   return (text: string | Buffer) => {
     const writeResult = res.write(text);
     if (!writeResult) {
-      readStream?.pause();
+      readStream?.pause?.();
     }
   };
 }

packages/service/core/app/schema.ts

@@ -1,3 +1,4 @@
+import { AppTypeMap } from '@fastgpt/global/core/app/constants';
 import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
@@ -31,7 +32,11 @@ const AppSchema = new Schema({
   type: {
     type: String,
     default: 'advanced',
-    enum: ['basic', 'advanced']
+    enum: Object.keys(AppTypeMap)
+  },
+  simpleTemplateId: {
+    type: String,
+    required: true
   },
   avatar: {
     type: String,
@@ -61,6 +66,7 @@ const AppSchema = new Schema({
 
 try {
   AppSchema.index({ updateTime: -1 });
+  AppSchema.index({ teamId: 1 });
 } catch (error) {
   console.log(error);
 }

packages/service/core/chat/chatItemSchema.ts

@@ -1,7 +1,7 @@
 import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type';
-import { ChatRoleMap, TaskResponseKeyEnum } from '@fastgpt/global/core/chat/constants';
+import { ChatRoleMap } from '@fastgpt/global/core/chat/constants';
 import { customAlphabet } from 'nanoid';
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
 import {
@@ -10,6 +10,7 @@ import {
 } from '@fastgpt/global/support/user/team/constant';
 import { appCollectionName } from '../app/schema';
 import { userCollectionName } from '../../support/user/schema';
+import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants';
 
 const ChatItemSchema = new Schema({
   dataId: {
@@ -65,7 +66,7 @@ const ChatItemSchema = new Schema({
       a: String
     }
   },
-  [TaskResponseKeyEnum.responseData]: {
+  [ModuleOutputKeyEnum.responseData]: {
     type: Array,
     default: []
   }

packages/service/core/chat/chatSchema.ts

@@ -1,16 +1,13 @@
 import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d';
-import {
-  ChatRoleMap,
-  ChatSourceMap,
-  TaskResponseKeyEnum
-} from '@fastgpt/global/core/chat/constants';
+import { ChatRoleMap, ChatSourceMap } from '@fastgpt/global/core/chat/constants';
 import {
   TeamCollectionName,
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 import { appCollectionName } from '../app/schema';
+import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants';
 
 export const chatCollectionName = 'chat';
 
@@ -81,7 +78,7 @@ const ChatSchema = new Schema({
           type: String,
           default: ''
         },
-        [TaskResponseKeyEnum.responseData]: {
+        [ModuleOutputKeyEnum.responseData]: {
           type: Array,
           default: []
         }

packages/service/core/chat/utils.ts

@@ -3,6 +3,7 @@ import { ChatRoleEnum, IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants
 import { countMessagesTokens, countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
 import { adaptRole_Chat2Message } from '@fastgpt/global/core/chat/adapt';
 import type { ChatCompletionContentPart } from '@fastgpt/global/core/ai/type.d';
+import axios from 'axios';
 
 /* slice chat context by tokens */
 export function ChatContextFilter({
@@ -81,11 +82,13 @@ export function ChatContextFilter({
             }
         ]
  */
-export function formatStr2ChatContent(str: string) {
+export async function formatStr2ChatContent(str: string) {
   const content: ChatCompletionContentPart[] = [];
   let lastIndex = 0;
   const regex = new RegExp(`\`\`\`(${IMG_BLOCK_KEY})\\n([\\s\\S]*?)\`\`\``, 'g');
 
+  const imgKey: 'image_url' = 'image_url';
+
   let match;
 
   while ((match = regex.exec(str)) !== null) {
@@ -115,7 +118,7 @@ export function formatStr2ChatContent(str: string) {
 
       content.push(
         ...jsonLines.map((item) => ({
-          type: 'image_url' as any,
+          type: imgKey,
           image_url: {
             url: item.src
           }
@@ -148,5 +151,18 @@ export function formatStr2ChatContent(str: string) {
   if (content.length === 1 && content[0].type === 'text') {
     return content[0].text;
   }
+
+  if (!content) return null;
+  // load img to base64
+  for await (const item of content) {
+    if (item.type === imgKey && item[imgKey]?.url) {
+      const response = await axios.get(item[imgKey].url, {
+        responseType: 'arraybuffer'
+      });
+      const base64 = Buffer.from(response.data).toString('base64');
+      item[imgKey].url = `data:${response.headers['content-type']};base64,${base64}`;
+    }
+  }
+
   return content ? content : null;
 }

packages/service/core/dataset/collection/controller.ts

@@ -0,0 +1,73 @@
+import {
+  DatasetCollectionTrainingModeEnum,
+  DatasetCollectionTypeEnum
+} from '@fastgpt/global/core/dataset/constant';
+import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
+import { MongoDatasetCollection } from './schema';
+
+export async function createOneCollection({
+  name,
+  parentId,
+  datasetId,
+  type,
+  trainingType = DatasetCollectionTrainingModeEnum.manual,
+  chunkSize = 0,
+  fileId,
+  rawLink,
+  teamId,
+  tmbId,
+  metadata = {}
+}: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
+  const { _id } = await MongoDatasetCollection.create({
+    name,
+    teamId,
+    tmbId,
+    datasetId,
+    parentId: parentId || null,
+    type,
+    trainingType,
+    chunkSize,
+    fileId,
+    rawLink,
+    metadata
+  });
+
+  // create default collection
+  if (type === DatasetCollectionTypeEnum.folder) {
+    await createDefaultCollection({
+      datasetId,
+      parentId: _id,
+      teamId,
+      tmbId
+    });
+  }
+
+  return _id;
+}
+
+// create default collection
+export function createDefaultCollection({
+  name = '手动录入',
+  datasetId,
+  parentId,
+  teamId,
+  tmbId
+}: {
+  name?: '手动录入' | '手动标注';
+  datasetId: string;
+  parentId?: string;
+  teamId: string;
+  tmbId: string;
+}) {
+  return MongoDatasetCollection.create({
+    name,
+    teamId,
+    tmbId,
+    datasetId,
+    parentId,
+    type: DatasetCollectionTypeEnum.virtual,
+    trainingType: DatasetCollectionTrainingModeEnum.manual,
+    chunkSize: 0,
+    updateTime: new Date('2099')
+  });
+}

packages/service/core/dataset/collection/schema.ts

@@ -1,7 +1,10 @@
 import { connectionMongo, type Model } from '../../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
-import { DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetCollectionTrainingTypeMap,
+  DatasetCollectionTypeMap
+} from '@fastgpt/global/core/dataset/constant';
 import { DatasetCollectionName } from '../schema';
 import {
   TeamCollectionName,
@@ -36,39 +39,49 @@ const DatasetCollectionSchema = new Schema({
     ref: DatasetCollectionName,
     required: true
   },
-  name: {
-    type: String,
-    required: true
-  },
   type: {
     type: String,
     enum: Object.keys(DatasetCollectionTypeMap),
     required: true
   },
+
+  name: {
+    type: String,
+    required: true
+  },
+  createTime: {
+    type: Date,
+    default: () => new Date()
+  },
   updateTime: {
     type: Date,
     default: () => new Date()
   },
+  trainingType: {
+    type: String,
+    enum: Object.keys(DatasetCollectionTrainingTypeMap),
+    required: true
+  },
+  chunkSize: {
+    type: Number,
+    required: true
+  },
+  fileId: {
+    type: Schema.Types.ObjectId,
+    ref: 'dataset.files'
+  },
+  rawLink: {
+    type: String
+  },
   metadata: {
-    type: {
-      fileId: {
-        type: Schema.Types.ObjectId,
-        ref: 'dataset.files'
-      },
-      rawLink: {
-        type: String
-      },
-      // 451 初始化
-      pgCollectionId: {
-        type: String
-      }
-    },
+    type: Object,
     default: {}
   }
 });
 
 try {
   DatasetCollectionSchema.index({ datasetId: 1 });
+  DatasetCollectionSchema.index({ datasetId: 1, parentId: 1 });
   DatasetCollectionSchema.index({ updateTime: -1 });
 } catch (error) {
   console.log(error);

packages/service/core/dataset/data/controller.ts

@@ -0,0 +1,75 @@
+import { MongoDatasetData } from './schema';
+import { deletePgDataById } from './pg';
+import { MongoDatasetTraining } from '../training/schema';
+import { delFileById } from '../../../common/file/gridfs/controller';
+import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { MongoDatasetCollection } from '../collection/schema';
+import { delDatasetFiles } from '../file/controller';
+import { delay } from '@fastgpt/global/common/system/utils';
+
+/* delete all data by datasetIds */
+export async function delDatasetRelevantData({ datasetIds }: { datasetIds: string[] }) {
+  datasetIds = datasetIds.map((item) => String(item));
+
+  // delete training data(There could be a training mission)
+  await MongoDatasetTraining.deleteMany({
+    datasetId: { $in: datasetIds }
+  });
+
+  // delete related files
+  await Promise.all(datasetIds.map((id) => delDatasetFiles({ datasetId: id })));
+
+  await delay(1000);
+
+  // delete pg data
+  await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`);
+  // delete dataset.datas
+  await MongoDatasetData.deleteMany({ datasetId: { $in: datasetIds } });
+
+  // delete collections
+  await MongoDatasetCollection.deleteMany({
+    datasetId: { $in: datasetIds }
+  });
+}
+/**
+ * delete all data by collectionIds
+ */
+export async function delCollectionRelevantData({
+  collectionIds,
+  fileIds
+}: {
+  collectionIds: string[];
+  fileIds: string[];
+}) {
+  collectionIds = collectionIds.map((item) => String(item));
+  const filterFileIds = fileIds.filter(Boolean);
+
+  // delete training data
+  await MongoDatasetTraining.deleteMany({
+    collectionId: { $in: collectionIds }
+  });
+
+  // delete file
+  await Promise.all(
+    filterFileIds.map((fileId) => {
+      return delFileById({
+        bucketName: BucketNameEnum.dataset,
+        fileId
+      });
+    })
+  );
+
+  await delay(1000);
+
+  // delete pg data
+  await deletePgDataById(`collection_id IN ('${collectionIds.join("','")}')`);
+  // delete dataset.datas
+  await MongoDatasetData.deleteMany({ collectionId: { $in: collectionIds } });
+}
+/**
+ * delete one data by mongoDataId
+ */
+export async function delDatasetDataByDataId(mongoDataId: string) {
+  await deletePgDataById(['data_id', mongoDataId]);
+  await MongoDatasetData.findByIdAndDelete(mongoDataId);
+}

packages/service/core/dataset/data/pg.ts

@@ -0,0 +1,28 @@
+import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
+import { delay } from '@fastgpt/global/common/system/utils';
+import { PgClient } from '../../../common/pg';
+
+export async function deletePgDataById(
+  where: ['id' | 'dataset_id' | 'collection_id' | 'data_id', string] | string
+) {
+  let retry = 2;
+  async function deleteData(): Promise<any> {
+    try {
+      await PgClient.delete(PgDatasetTableName, {
+        where: [where]
+      });
+    } catch (error) {
+      if (--retry < 0) {
+        return Promise.reject(error);
+      }
+      await delay(500);
+      return deleteData();
+    }
+  }
+
+  await deleteData();
+
+  return {
+    tokenLen: 0
+  };
+}

packages/service/core/dataset/data/schema.ts

@@ -43,6 +43,10 @@ const DatasetDataSchema = new Schema({
     type: String,
     default: ''
   },
+  fullTextToken: {
+    type: String,
+    default: ''
+  },
   indexes: {
     type: [
       {
@@ -66,13 +70,28 @@ const DatasetDataSchema = new Schema({
       }
     ],
     default: []
+  },
+  // metadata
+  updateTime: {
+    type: Date,
+    default: () => new Date()
+  },
+  chunkIndex: {
+    type: Number,
+    default: 0
+  },
+  inited: {
+    type: Boolean
   }
 });
 
 try {
-  DatasetDataSchema.index({ userId: 1 });
+  DatasetDataSchema.index({ teamId: 1 });
   DatasetDataSchema.index({ datasetId: 1 });
   DatasetDataSchema.index({ collectionId: 1 });
+  // full text index
+  DatasetDataSchema.index({ datasetId: 1, fullTextToken: 'text' });
+  DatasetDataSchema.index({ inited: 1 });
 } catch (error) {
   console.log(error);
 }

packages/service/core/dataset/schema.ts

@@ -1,7 +1,11 @@
 import { connectionMongo, type Model } from '../../common/mongo';
 const { Schema, model, models } = connectionMongo;
 import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
-import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constant';
+import {
+  DatasetStatusEnum,
+  DatasetStatusMap,
+  DatasetTypeMap
+} from '@fastgpt/global/core/dataset/constant';
 import {
   TeamCollectionName,
   TeamMemberCollectionName
@@ -31,9 +35,16 @@ const DatasetSchema = new Schema({
     ref: TeamMemberCollectionName,
     required: true
   },
-  updateTime: {
-    type: Date,
-    default: () => new Date()
+  type: {
+    type: String,
+    enum: Object.keys(DatasetTypeMap),
+    required: true,
+    default: 'dataset'
+  },
+  status: {
+    type: String,
+    enum: Object.keys(DatasetStatusMap),
+    default: DatasetStatusEnum.active
   },
   avatar: {
     type: String,
@@ -43,6 +54,10 @@ const DatasetSchema = new Schema({
     type: String,
     required: true
   },
+  updateTime: {
+    type: Date,
+    default: () => new Date()
+  },
   vectorModel: {
     type: String,
     required: true,
@@ -53,24 +68,26 @@ const DatasetSchema = new Schema({
     required: true,
     default: 'gpt-3.5-turbo-16k'
   },
-  type: {
+  intro: {
     type: String,
-    enum: Object.keys(DatasetTypeMap),
-    required: true,
-    default: 'dataset'
-  },
-  tags: {
-    type: [String],
-    default: [],
-    set(val: string | string[]) {
-      if (Array.isArray(val)) return val;
-      return val.split(' ').filter((item) => item);
-    }
+    default: ''
   },
   permission: {
     type: String,
     enum: Object.keys(PermissionTypeMap),
     default: PermissionTypeEnum.private
+  },
+  websiteConfig: {
+    type: {
+      url: {
+        type: String,
+        required: true
+      },
+      selector: {
+        type: String,
+        default: 'body'
+      }
+    }
   }
 });
 

packages/service/core/dataset/training/schema.ts

@@ -75,6 +75,10 @@ const TrainingDataSchema = new Schema({
     type: String,
     default: ''
   },
+  chunkIndex: {
+    type: Number,
+    default: 0
+  },
   indexes: {
     type: [
       {

packages/service/core/plugin/controller.ts

@@ -1,40 +1,100 @@
 import { MongoPlugin } from './schema';
 import { FlowModuleTemplateType } from '@fastgpt/global/core/module/type';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
-import { formatPluginIOModules } from '@fastgpt/global/core/module/utils';
+import { formatPluginToPreviewModule } from '@fastgpt/global/core/module/utils';
+import { PluginType2TemplateTypeMap, PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
+import type { PluginTemplateType } from '@fastgpt/global/core/plugin/type.d';
 
-/* plugin templates */
-export async function getUserPlugins2Templates({
-  teamId
-}: {
-  teamId: string;
-}): Promise<FlowModuleTemplateType[]> {
-  const plugins = await MongoPlugin.find({ teamId }).lean();
+/* 
+  plugin id rule:
+  personal: id
+  community: community-id
+  commercial: commercial-id
+*/
 
-  return plugins.map((plugin) => ({
-    id: String(plugin._id),
-    flowType: FlowNodeTypeEnum.pluginModule,
-    logo: plugin.avatar,
-    name: plugin.name,
-    description: plugin.intro,
-    intro: plugin.intro,
-    showStatus: false,
-    inputs: [],
-    outputs: []
-  }));
+export async function splitCombinePluginId(id: string) {
+  const splitRes = id.split('-');
+  if (splitRes.length === 1 && id.length === 24) {
+    return {
+      type: PluginTypeEnum.personal,
+      pluginId: id
+    };
+  }
+
+  const [type, pluginId] = id.split('-') as [`${PluginTypeEnum}`, string];
+  if (!type || !pluginId) return Promise.reject('pluginId not found');
+
+  return { type, pluginId: id };
 }
-/* one plugin 2 module detail */
-export async function getPluginModuleDetail({ id }: { id: string }) {
-  const plugin = await MongoPlugin.findById(id);
+/* format plugin modules to plugin preview module */
+export async function getPluginPreviewModule({
+  id
+}: {
+  id: string;
+}): Promise<FlowModuleTemplateType> {
+  // classify
+  const { type, pluginId } = await splitCombinePluginId(id);
+
+  const plugin = await (async () => {
+    if (type === PluginTypeEnum.community) {
+      return global.communityPlugins?.find((plugin) => plugin.id === pluginId);
+    }
+    if (type === PluginTypeEnum.personal) {
+      const item = await MongoPlugin.findById(id);
+      if (!item) return undefined;
+      return {
+        id: String(item._id),
+        name: item.name,
+        avatar: item.avatar,
+        intro: item.intro,
+        type: PluginTypeEnum.personal,
+        modules: item.modules
+      };
+    }
+  })();
+
   if (!plugin) return Promise.reject('plugin not found');
   return {
-    id: String(plugin._id),
+    id: plugin.id,
+    templateType: PluginType2TemplateTypeMap[plugin.type],
     flowType: FlowNodeTypeEnum.pluginModule,
-    logo: plugin.avatar,
+    avatar: plugin.avatar,
     name: plugin.name,
-    description: plugin.intro,
     intro: plugin.intro,
-    showStatus: false,
-    ...formatPluginIOModules(String(plugin._id), plugin.modules)
+    showStatus: true,
+    ...formatPluginToPreviewModule(plugin.id, plugin.modules)
+  };
+}
+
+export async function getPluginRuntimeById(id: string): Promise<PluginTemplateType> {
+  const { type, pluginId } = await splitCombinePluginId(id);
+
+  const plugin = await (async () => {
+    if (type === PluginTypeEnum.community) {
+      return global.communityPlugins?.find((plugin) => plugin.id === pluginId);
+    }
+    if (type === PluginTypeEnum.personal) {
+      const item = await MongoPlugin.findById(id);
+      if (!item) return undefined;
+      return {
+        id: String(item._id),
+        name: item.name,
+        avatar: item.avatar,
+        intro: item.intro,
+        type: PluginTypeEnum.personal,
+        modules: item.modules
+      };
+    }
+  })();
+
+  if (!plugin) return Promise.reject('plugin not found');
+
+  return {
+    id: plugin.id,
+    type: plugin.type,
+    name: plugin.name,
+    avatar: plugin.avatar,
+    intro: plugin.intro,
+    modules: plugin.modules
   };
 }

packages/service/core/plugin/schema.ts

@@ -6,7 +6,7 @@ import {
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 
-export const ModuleCollectionName = 'plugins';
+export const PluginCollectionName = 'plugins';
 
 const PluginSchema = new Schema({
   userId: {
@@ -52,5 +52,5 @@ try {
 }
 
 export const MongoPlugin: Model<PluginItemSchema> =
-  models[ModuleCollectionName] || model(ModuleCollectionName, PluginSchema);
+  models[PluginCollectionName] || model(PluginCollectionName, PluginSchema);
 MongoPlugin.syncIndexes();

packages/service/core/plugin/store/schema.ts

@@ -0,0 +1,31 @@
+import { connectionMongo, type Model } from '../../../common/mongo';
+const { Schema, model, models } = connectionMongo;
+import type { PluginItemSchema } from '@fastgpt/global/core/plugin/type.d';
+
+import { PluginCollectionName } from '../schema';
+
+export const ModuleCollectionName = 'plugins';
+
+const PluginStoreSchema = new Schema({
+  pluginId: {
+    type: Schema.Types.ObjectId,
+    ref: PluginCollectionName,
+    required: true
+  },
+  price: {
+    type: Number,
+    default: 0
+  },
+  updateTime: {
+    type: Date,
+    default: () => new Date()
+  },
+  modules: {
+    type: Array,
+    default: []
+  }
+});
+
+export const MongoPluginStore: Model<PluginItemSchema> =
+  models[ModuleCollectionName] || model(ModuleCollectionName, PluginStoreSchema);
+MongoPluginStore.syncIndexes();

packages/service/core/plugin/type.d.ts

@@ -0,0 +1,5 @@
+import { PluginTemplateType } from '@fastgpt/global/core/plugin/type.d';
+
+declare global {
+  var communityPlugins: PluginTemplateType[];
+}

packages/service/package.json

@@ -4,23 +4,23 @@
   "dependencies": {
     "@fastgpt/global": "workspace:*",
     "axios": "^1.5.1",
-    "nextjs-cors": "^2.1.2",
-    "next": "13.5.2",
     "cookie": "^0.5.0",
+    "encoding": "^0.1.13",
     "jsonwebtoken": "^9.0.2",
     "mongoose": "^7.0.2",
-    "winston": "^3.10.0",
-    "winston-mongodb": "^5.1.1",
-    "tunnel": "^0.0.6",
-    "encoding": "^0.1.13",
+    "nanoid": "^4.0.1",
+    "next": "13.5.2",
+    "nextjs-cors": "^2.1.2",
     "pg": "^8.10.0",
-    "nanoid": "^4.0.1"
+    "tunnel": "^0.0.6",
+    "winston": "^3.10.0",
+    "winston-mongodb": "^5.1.1"
   },
   "devDependencies": {
-    "@types/tunnel": "^0.0.4",
-    "@types/pg": "^8.6.6",
-    "@types/node": "^20.8.5",
     "@types/cookie": "^0.5.2",
-    "@types/jsonwebtoken": "^9.0.3"
+    "@types/jsonwebtoken": "^9.0.3",
+    "@types/node": "^20.8.5",
+    "@types/pg": "^8.6.6",
+    "@types/tunnel": "^0.0.4"
   }
 }

packages/service/support/permission/auth/chat.ts

@@ -1,6 +1,6 @@
 import { AuthResponseType } from '@fastgpt/global/support/permission/type';
 import { AuthModeType } from '../type';
-import type { ChatSchema, ChatWithAppSchema } from '@fastgpt/global/core/chat/type';
+import type { ChatWithAppSchema } from '@fastgpt/global/core/chat/type';
 import { parseHeaderCert } from '../controller';
 import { MongoChat } from '../../../core/chat/chatSchema';
 import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
@@ -16,7 +16,7 @@ export async function authChat({
   chatId: string;
 }): Promise<
   AuthResponseType & {
-    chat: ChatSchema;
+    chat: ChatWithAppSchema;
   }
 > {
   const { userId, teamId, tmbId } = await parseHeaderCert(props);

packages/service/support/permission/auth/dataset.ts

@@ -15,19 +15,17 @@ import { getFileById } from '../../../common/file/gridfs/controller';
 import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import { getTeamInfoByTmbId } from '../../user/team/controller';
 
-export async function authDataset({
+export async function authDatasetByTmbId({
+  teamId,
+  tmbId,
   datasetId,
-  per = 'owner',
-  ...props
-}: AuthModeType & {
+  per
+}: {
+  teamId: string;
+  tmbId: string;
   datasetId: string;
-}): Promise<
-  AuthResponseType & {
-    dataset: DatasetSchemaType;
-  }
-> {
-  const result = await parseHeaderCert(props);
-  const { teamId, tmbId } = result;
+  per: AuthModeType['per'];
+}) {
   const { role } = await getTeamInfoByTmbId({ tmbId });
 
   const { dataset, isOwner, canWrite } = await (async () => {
@@ -58,6 +56,32 @@ export async function authDataset({
     return { dataset, isOwner, canWrite };
   })();
 
+  return {
+    dataset,
+    isOwner,
+    canWrite
+  };
+}
+export async function authDataset({
+  datasetId,
+  per = 'owner',
+  ...props
+}: AuthModeType & {
+  datasetId: string;
+}): Promise<
+  AuthResponseType & {
+    dataset: DatasetSchemaType;
+  }
+> {
+  const result = await parseHeaderCert(props);
+  const { teamId, tmbId } = result;
+  const { dataset, isOwner, canWrite } = await authDatasetByTmbId({
+    teamId,
+    tmbId,
+    datasetId,
+    per
+  });
+
   return {
     ...result,
     dataset,

packages/service/support/permission/auth/plugin.ts

@@ -6,6 +6,8 @@ import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
 import { MongoPlugin } from '../../../core/plugin/schema';
 import { PluginErrEnum } from '@fastgpt/global/common/error/code/plugin';
 import { PluginItemSchema } from '@fastgpt/global/core/plugin/type';
+import { splitCombinePluginId } from '../../../core/plugin/controller';
+import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
 
 export async function authPluginCrud({
   id,
@@ -54,3 +56,29 @@ export async function authPluginCrud({
     canWrite
   };
 }
+
+export async function authPluginCanUse({
+  id,
+  teamId,
+  tmbId
+}: {
+  id: string;
+  teamId: string;
+  tmbId: string;
+}) {
+  const { type, pluginId } = await splitCombinePluginId(id);
+
+  if (type === PluginTypeEnum.community) {
+    return true;
+  }
+
+  if (type === PluginTypeEnum.personal) {
+    const { role } = await getTeamInfoByTmbId({ tmbId });
+    const plugin = await MongoPlugin.findOne({ _id: pluginId, teamId });
+    if (!plugin) {
+      return Promise.reject(PluginErrEnum.unExist);
+    }
+  }
+
+  return true;
+}

packages/service/support/permission/controller.ts

@@ -105,14 +105,13 @@ export async function parseHeaderCert({
     };
   }
   // root user
-  async function parseRootKey(rootKey?: string, userId = '') {
+  async function parseRootKey(rootKey?: string) {
     if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
       return Promise.reject(ERROR_ENUM.unAuthorization);
     }
-    return userId;
   }
 
-  const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers ||
+  const { cookie, token, apikey, rootkey, authorization } = (req.headers ||
     {}) as ReqHeaderAuthType;
 
   const { uid, teamId, tmbId, appId, openApiKey, authType } = await (async () => {
@@ -129,9 +128,10 @@ export async function parseHeaderCert({
       };
     }
     if (authRoot && rootkey) {
+      await parseRootKey(rootkey);
       // root user
       return {
-        uid: await parseRootKey(rootkey, userid),
+        uid: '',
         teamId: '',
         tmbId: '',
         appId: '',

pnpm-lock.yaml

@@ -1,4 +1,4 @@
-lockfileVersion: '6.0'
+lockfileVersion: '6.1'
 
 settings:
   autoInstallPeers: true
@@ -45,6 +45,9 @@ importers:
       axios:
         specifier: ^1.5.1
         version: registry.npmmirror.com/axios@1.5.1
+      cheerio:
+        specifier: 1.0.0-rc.12
+        version: registry.npmmirror.com/cheerio@1.0.0-rc.12
       dayjs:
         specifier: ^1.11.7
         version: registry.npmmirror.com/dayjs@1.11.10
@@ -54,6 +57,9 @@ importers:
       js-tiktoken:
         specifier: ^1.0.7
         version: registry.npmmirror.com/js-tiktoken@1.0.7
+      node-html-markdown:
+        specifier: ^1.3.0
+        version: registry.npmmirror.com/node-html-markdown@1.3.0
       openai:
         specifier: ^4.16.1
         version: registry.npmmirror.com/openai@4.16.1(encoding@0.1.13)
@@ -64,6 +70,9 @@ importers:
       '@types/node':
         specifier: ^20.8.5
         version: registry.npmmirror.com/@types/node@20.8.7
+      '@types/turndown':
+        specifier: ^5.0.4
+        version: registry.npmmirror.com/@types/turndown@5.0.4
 
   packages/service:
     dependencies:
@@ -161,9 +170,9 @@ importers:
       '@fastgpt/web':
         specifier: workspace:*
         version: link:../../packages/web
-      '@mozilla/readability':
-        specifier: ^0.4.4
-        version: registry.npmmirror.com/@mozilla/readability@0.4.4
+      '@node-rs/jieba':
+        specifier: ^1.7.2
+        version: registry.npmmirror.com/@node-rs/jieba@1.7.2
       '@tanstack/react-query':
         specifier: ^4.24.10
         version: registry.npmmirror.com/@tanstack/react-query@4.36.1(react-dom@18.2.0)(react@18.2.0)
@@ -206,9 +215,6 @@ importers:
       jschardet:
         specifier: ^3.0.0
         version: registry.npmmirror.com/jschardet@3.0.0
-      jsdom:
-        specifier: ^22.1.0
-        version: registry.npmmirror.com/jsdom@22.1.0
       jsonwebtoken:
         specifier: ^9.0.2
         version: registry.npmmirror.com/jsonwebtoken@9.0.2
@@ -216,7 +222,7 @@ importers:
         specifier: ^4.17.21
         version: registry.npmmirror.com/lodash@4.17.21
       mammoth:
-        specifier: ^1.5.1
+        specifier: ^1.6.0
         version: registry.npmmirror.com/mammoth@1.6.0
       mermaid:
         specifier: ^10.2.3
@@ -297,9 +303,6 @@ importers:
       '@types/js-cookie':
         specifier: ^3.0.3
         version: registry.npmmirror.com/@types/js-cookie@3.0.5
-      '@types/jsdom':
-        specifier: ^21.1.1
-        version: registry.npmmirror.com/@types/jsdom@21.1.4
       '@types/jsonwebtoken':
         specifier: ^9.0.3
         version: registry.npmmirror.com/@types/jsonwebtoken@9.0.4
@@ -3572,13 +3575,6 @@ packages:
     dev: false
     optional: true
 
-  registry.npmmirror.com/@mozilla/readability@0.4.4:
-    resolution: {integrity: sha512-MCgZyANpJ6msfvVMi6+A0UAsvZj//4OHREYUB9f2087uXHVoU+H+SWhuihvb1beKpM323bReQPRio0WNk2+V6g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@mozilla/readability/-/readability-0.4.4.tgz}
-    name: '@mozilla/readability'
-    version: 0.4.4
-    engines: {node: '>=14.0.0'}
-    dev: false
-
   registry.npmmirror.com/@next/env@13.5.2:
     resolution: {integrity: sha512-dUseBIQVax+XtdJPzhwww4GetTjlkRSsXeQnisIJWBaHsnxYcN2RGzsPHi58D6qnkATjnhuAtQTJmR1hKYQQPg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@next/env/-/env-13.5.2.tgz}
     name: '@next/env'
@@ -3686,6 +3682,174 @@ packages:
     requiresBuild: true
     optional: true
 
+  registry.npmmirror.com/@node-rs/jieba-android-arm-eabi@1.7.2:
+    resolution: {integrity: sha512-FyDHRNSRIHOQO7S6Q4RwuGffnnnuNwaXPH7K8WqSzifEY+zFIaSPcNqrZHrnqyeXc4JiYpBIHeP+0Mkf1kIGRA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-android-arm-eabi/-/jieba-android-arm-eabi-1.7.2.tgz}
+    name: '@node-rs/jieba-android-arm-eabi'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [arm]
+    os: [android]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-android-arm64@1.7.2:
+    resolution: {integrity: sha512-z0UEZCGrAX/IiarhuDMsEIDZBS77UZv4SQyL/J48yrsbWKbb2lJ1vCrYxXIWqwp6auXHEu4r1O/pMriDAcEnPg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-android-arm64/-/jieba-android-arm64-1.7.2.tgz}
+    name: '@node-rs/jieba-android-arm64'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [android]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-darwin-arm64@1.7.2:
+    resolution: {integrity: sha512-M2cHIWRaaOmXGKy446SH2+Y2PzREaI2oYznPbg55wYEdioUp01YS/2WRG8CaoCKEj0aUocA7MFM2vVcoIAsbQw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-darwin-arm64/-/jieba-darwin-arm64-1.7.2.tgz}
+    name: '@node-rs/jieba-darwin-arm64'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [darwin]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-darwin-x64@1.7.2:
+    resolution: {integrity: sha512-euDawBU2FxB0CGTR803BA6WABsiicIrqa61z2AFFDPkJCDrauEM0jbMg3GDKLAvbaLbZ1Etu3QNN5xyroqp4Qw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-darwin-x64/-/jieba-darwin-x64-1.7.2.tgz}
+    name: '@node-rs/jieba-darwin-x64'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [darwin]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-freebsd-x64@1.7.2:
+    resolution: {integrity: sha512-vXCaYxPb90d/xTBVG+ZZXrFLXsO2719pZSyiZCL2tey+UY28U7MOoK6394Wwmf0FCB/eRTQMCKjVIUDi+IRMUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-freebsd-x64/-/jieba-freebsd-x64-1.7.2.tgz}
+    name: '@node-rs/jieba-freebsd-x64'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-linux-arm-gnueabihf@1.7.2:
+    resolution: {integrity: sha512-HTep79XlJYO3KRYZ2kJChG9HnYr1DKSQTB+HEYWKLK0ifphqybcxGNLAdH0S4dViG2ciD0+iN/refgtqZEidpw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-linux-arm-gnueabihf/-/jieba-linux-arm-gnueabihf-1.7.2.tgz}
+    name: '@node-rs/jieba-linux-arm-gnueabihf'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [arm]
+    os: [linux]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-linux-arm64-gnu@1.7.2:
+    resolution: {integrity: sha512-P8QJdQydOVewL1MIqYiRpI7LOfrRQag+p4/hwExe+YXH8C7DOrR8rWJD/7XNRTbpOimlHq1UN/e+ZzhxQF/cLw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-linux-arm64-gnu/-/jieba-linux-arm64-gnu-1.7.2.tgz}
+    name: '@node-rs/jieba-linux-arm64-gnu'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [linux]
+    libc: [glibc]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-linux-arm64-musl@1.7.2:
+    resolution: {integrity: sha512-WjnN0hmDvTXb2h3hMW5VnUGkK1xaqhs+WHfMMilau55+YN+YOYALKZ0TeBY4BapClLuBx54wqwmBX+B4hAXunQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-linux-arm64-musl/-/jieba-linux-arm64-musl-1.7.2.tgz}
+    name: '@node-rs/jieba-linux-arm64-musl'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [linux]
+    libc: [musl]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-linux-x64-gnu@1.7.2:
+    resolution: {integrity: sha512-gBXds/DwNSA6lNUxJjL6WIaNT6pnlM5juUgV/krLLkBJ8vXpOrQ07p0rrK1tnigz9b20xhsHaFRSwED1Y8zeXw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-linux-x64-gnu/-/jieba-linux-x64-gnu-1.7.2.tgz}
+    name: '@node-rs/jieba-linux-x64-gnu'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [linux]
+    libc: [glibc]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-linux-x64-musl@1.7.2:
+    resolution: {integrity: sha512-tNVD3SMuG5zAj7+bLS2Enio3zR7BPxi3PhQtpQ+Hv83jajIcN46QQ0EdoMFz/aB+hkQ9PlLAstu+VREFegs5EA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-linux-x64-musl/-/jieba-linux-x64-musl-1.7.2.tgz}
+    name: '@node-rs/jieba-linux-x64-musl'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [linux]
+    libc: [musl]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-win32-arm64-msvc@1.7.2:
+    resolution: {integrity: sha512-/e1iQ0Dh02lGPNCYTU/H3cfIsWydaGRzZ3TDj6GfWrxkWqXORL98x/VJ/C/uKLpc7GSLLd9ygyZG7SOAfKe2tA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-win32-arm64-msvc/-/jieba-win32-arm64-msvc-1.7.2.tgz}
+    name: '@node-rs/jieba-win32-arm64-msvc'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [arm64]
+    os: [win32]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-win32-ia32-msvc@1.7.2:
+    resolution: {integrity: sha512-cYjA6YUiOwtuEzWErvwMMt/RETNWQDLcmAaiHA8ohsa6c0eB0kRJlQCc683tlaczZxqroY/7C9mxgJNGvoGRbw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-win32-ia32-msvc/-/jieba-win32-ia32-msvc-1.7.2.tgz}
+    name: '@node-rs/jieba-win32-ia32-msvc'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [ia32]
+    os: [win32]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba-win32-x64-msvc@1.7.2:
+    resolution: {integrity: sha512-2M+Um3woFF17sa8VBYQQ6E5PNMe9Kf9fdzmeDh/GzuNHXlxW4LyK9VTV8zchIv/bDNAR5Z85kfW4wASULUxvFQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba-win32-x64-msvc/-/jieba-win32-x64-msvc-1.7.2.tgz}
+    name: '@node-rs/jieba-win32-x64-msvc'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    cpu: [x64]
+    os: [win32]
+    requiresBuild: true
+    dev: false
+    optional: true
+
+  registry.npmmirror.com/@node-rs/jieba@1.7.2:
+    resolution: {integrity: sha512-zGto08NDU+KWm670qVHYGTb0YTEJ0A97dwH3WCnnhyRYMqTbOXKC6OwTc/cjzfSJP1UDBSar9Ug9BlmWmEThWg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@node-rs/jieba/-/jieba-1.7.2.tgz}
+    name: '@node-rs/jieba'
+    version: 1.7.2
+    engines: {node: '>= 10'}
+    optionalDependencies:
+      '@node-rs/jieba-android-arm-eabi': registry.npmmirror.com/@node-rs/jieba-android-arm-eabi@1.7.2
+      '@node-rs/jieba-android-arm64': registry.npmmirror.com/@node-rs/jieba-android-arm64@1.7.2
+      '@node-rs/jieba-darwin-arm64': registry.npmmirror.com/@node-rs/jieba-darwin-arm64@1.7.2
+      '@node-rs/jieba-darwin-x64': registry.npmmirror.com/@node-rs/jieba-darwin-x64@1.7.2
+      '@node-rs/jieba-freebsd-x64': registry.npmmirror.com/@node-rs/jieba-freebsd-x64@1.7.2
+      '@node-rs/jieba-linux-arm-gnueabihf': registry.npmmirror.com/@node-rs/jieba-linux-arm-gnueabihf@1.7.2
+      '@node-rs/jieba-linux-arm64-gnu': registry.npmmirror.com/@node-rs/jieba-linux-arm64-gnu@1.7.2
+      '@node-rs/jieba-linux-arm64-musl': registry.npmmirror.com/@node-rs/jieba-linux-arm64-musl@1.7.2
+      '@node-rs/jieba-linux-x64-gnu': registry.npmmirror.com/@node-rs/jieba-linux-x64-gnu@1.7.2
+      '@node-rs/jieba-linux-x64-musl': registry.npmmirror.com/@node-rs/jieba-linux-x64-musl@1.7.2
+      '@node-rs/jieba-win32-arm64-msvc': registry.npmmirror.com/@node-rs/jieba-win32-arm64-msvc@1.7.2
+      '@node-rs/jieba-win32-ia32-msvc': registry.npmmirror.com/@node-rs/jieba-win32-ia32-msvc@1.7.2
+      '@node-rs/jieba-win32-x64-msvc': registry.npmmirror.com/@node-rs/jieba-win32-x64-msvc@1.7.2
+    dev: false
+
   registry.npmmirror.com/@nodelib/fs.scandir@2.1.5:
     resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz}
     name: '@nodelib/fs.scandir'
@@ -4077,13 +4241,6 @@ packages:
       use-sync-external-store: registry.npmmirror.com/use-sync-external-store@1.2.0(react@18.2.0)
     dev: false
 
-  registry.npmmirror.com/@tootallnate/once@2.0.0:
-    resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@tootallnate/once/-/once-2.0.0.tgz}
-    name: '@tootallnate/once'
-    version: 2.0.0
-    engines: {node: '>= 10'}
-    dev: false
-
   registry.npmmirror.com/@trysound/sax@0.2.0:
     resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@trysound/sax/-/sax-0.2.0.tgz}
     name: '@trysound/sax'
@@ -4447,16 +4604,6 @@ packages:
     version: 3.0.5
     dev: true
 
-  registry.npmmirror.com/@types/jsdom@21.1.4:
-    resolution: {integrity: sha512-NzAMLEV0KQ4cBaDx3Ls8VfJUElyDUm1xrtYRmcMK0gF8L5xYbujFVaQlJ50yinQ/d47j2rEP1XUzkiYrw4YRFA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/jsdom/-/jsdom-21.1.4.tgz}
-    name: '@types/jsdom'
-    version: 21.1.4
-    dependencies:
-      '@types/node': registry.npmmirror.com/@types/node@20.8.7
-      '@types/tough-cookie': registry.npmmirror.com/@types/tough-cookie@4.0.4
-      parse5: registry.npmmirror.com/parse5@7.1.2
-    dev: true
-
   registry.npmmirror.com/@types/json5@0.0.29:
     resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/json5/-/json5-0.0.29.tgz}
     name: '@types/json5'
@@ -4655,12 +4802,6 @@ packages:
       '@types/node': registry.npmmirror.com/@types/node@20.8.7
     dev: true
 
-  registry.npmmirror.com/@types/tough-cookie@4.0.4:
-    resolution: {integrity: sha512-95Sfz4nvMAb0Nl9DTxN3j64adfwfbBPEYq14VN7zT5J5O2M9V6iZMIIQU1U+pJyl9agHYHNCqhCXgyEtIRRa5A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/tough-cookie/-/tough-cookie-4.0.4.tgz}
-    name: '@types/tough-cookie'
-    version: 4.0.4
-    dev: true
-
   registry.npmmirror.com/@types/triple-beam@1.3.4:
     resolution: {integrity: sha512-HlJjF3wxV4R2VQkFpKe0YqJLilYNgtRtsqqZtby7RkVsSs+i+vbyzjtUwpFEdUCKcrGzCiEJE7F/0mKjh0sunA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/triple-beam/-/triple-beam-1.3.4.tgz}
     name: '@types/triple-beam'
@@ -4675,6 +4816,12 @@ packages:
       '@types/node': registry.npmmirror.com/@types/node@20.8.7
     dev: true
 
+  registry.npmmirror.com/@types/turndown@5.0.4:
+    resolution: {integrity: sha512-28GI33lCCkU4SGH1GvjDhFgOVr+Tym4PXGBIU1buJUa6xQolniPArtUT+kv42RR2N9MsMLInkr904Aq+ESHBJg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/turndown/-/turndown-5.0.4.tgz}
+    name: '@types/turndown'
+    version: 5.0.4
+    dev: true
+
   registry.npmmirror.com/@types/unist@2.0.9:
     resolution: {integrity: sha512-zC0iXxAv1C1ERURduJueYzkzZ2zaGyc+P2c95hgkikHPr3z8EdUZOlgEQ5X0DRmwDZn+hekycQnoeiiRVrmilQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/unist/-/unist-2.0.9.tgz}
     name: '@types/unist'
@@ -4796,12 +4943,6 @@ packages:
       '@zag-js/dom-query': registry.npmmirror.com/@zag-js/dom-query@0.16.0
     dev: false
 
-  registry.npmmirror.com/abab@2.0.6:
-    resolution: {integrity: sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abab/-/abab-2.0.6.tgz}
-    name: abab
-    version: 2.0.6
-    dev: false
-
   registry.npmmirror.com/abort-controller@3.0.0:
     resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/abort-controller/-/abort-controller-3.0.0.tgz}
     name: abort-controller
@@ -4830,17 +4971,6 @@ packages:
     hasBin: true
     dev: true
 
-  registry.npmmirror.com/agent-base@6.0.2:
-    resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/agent-base/-/agent-base-6.0.2.tgz}
-    name: agent-base
-    version: 6.0.2
-    engines: {node: '>= 6.0.0'}
-    dependencies:
-      debug: registry.npmmirror.com/debug@4.3.4
-    transitivePeerDependencies:
-      - supports-color
-    dev: false
-
   registry.npmmirror.com/agentkeepalive@4.5.0:
     resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz}
     name: agentkeepalive
@@ -5264,7 +5394,6 @@ packages:
     resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz}
     name: boolbase
     version: 1.0.0
-    dev: true
 
   registry.npmmirror.com/brace-expansion@1.1.11:
     resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz}
@@ -5549,6 +5678,34 @@ packages:
       get-func-name: registry.npmmirror.com/get-func-name@2.0.2
     dev: true
 
+  registry.npmmirror.com/cheerio-select@2.1.0:
+    resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cheerio-select/-/cheerio-select-2.1.0.tgz}
+    name: cheerio-select
+    version: 2.1.0
+    dependencies:
+      boolbase: registry.npmmirror.com/boolbase@1.0.0
+      css-select: registry.npmmirror.com/css-select@5.1.0
+      css-what: registry.npmmirror.com/css-what@6.1.0
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      domutils: registry.npmmirror.com/domutils@3.1.0
+    dev: false
+
+  registry.npmmirror.com/cheerio@1.0.0-rc.12:
+    resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cheerio/-/cheerio-1.0.0-rc.12.tgz}
+    name: cheerio
+    version: 1.0.0-rc.12
+    engines: {node: '>= 6'}
+    dependencies:
+      cheerio-select: registry.npmmirror.com/cheerio-select@2.1.0
+      dom-serializer: registry.npmmirror.com/dom-serializer@2.0.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      domutils: registry.npmmirror.com/domutils@3.1.0
+      htmlparser2: registry.npmmirror.com/htmlparser2@8.0.2
+      parse5: registry.npmmirror.com/parse5@7.1.2
+      parse5-htmlparser2-tree-adapter: registry.npmmirror.com/parse5-htmlparser2-tree-adapter@7.0.0
+    dev: false
+
   registry.npmmirror.com/chokidar@3.5.3:
     resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/chokidar/-/chokidar-3.5.3.tgz}
     name: chokidar
@@ -5937,6 +6094,18 @@ packages:
       nth-check: registry.npmmirror.com/nth-check@2.1.1
     dev: true
 
+  registry.npmmirror.com/css-select@5.1.0:
+    resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/css-select/-/css-select-5.1.0.tgz}
+    name: css-select
+    version: 5.1.0
+    dependencies:
+      boolbase: registry.npmmirror.com/boolbase@1.0.0
+      css-what: registry.npmmirror.com/css-what@6.1.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      domutils: registry.npmmirror.com/domutils@3.1.0
+      nth-check: registry.npmmirror.com/nth-check@2.1.1
+    dev: false
+
   registry.npmmirror.com/css-tree@1.1.3:
     resolution: {integrity: sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/css-tree/-/css-tree-1.1.3.tgz}
     name: css-tree
@@ -5952,7 +6121,6 @@ packages:
     name: css-what
     version: 6.1.0
     engines: {node: '>= 6'}
-    dev: true
 
   registry.npmmirror.com/csso@4.2.0:
     resolution: {integrity: sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/csso/-/csso-4.2.0.tgz}
@@ -5963,15 +6131,6 @@ packages:
       css-tree: registry.npmmirror.com/css-tree@1.1.3
     dev: true
 
-  registry.npmmirror.com/cssstyle@3.0.0:
-    resolution: {integrity: sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cssstyle/-/cssstyle-3.0.0.tgz}
-    name: cssstyle
-    version: 3.0.0
-    engines: {node: '>=14'}
-    dependencies:
-      rrweb-cssom: registry.npmmirror.com/rrweb-cssom@0.6.0
-    dev: false
-
   registry.npmmirror.com/csstype@3.1.2:
     resolution: {integrity: sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/csstype/-/csstype-3.1.2.tgz}
     name: csstype
@@ -6368,17 +6527,6 @@ packages:
     version: 1.0.8
     dev: true
 
-  registry.npmmirror.com/data-urls@4.0.0:
-    resolution: {integrity: sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/data-urls/-/data-urls-4.0.0.tgz}
-    name: data-urls
-    version: 4.0.0
-    engines: {node: '>=14'}
-    dependencies:
-      abab: registry.npmmirror.com/abab@2.0.6
-      whatwg-mimetype: registry.npmmirror.com/whatwg-mimetype@3.0.0
-      whatwg-url: registry.npmmirror.com/whatwg-url@12.0.1
-    dev: false
-
   registry.npmmirror.com/date-fns@2.30.0:
     resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/date-fns/-/date-fns-2.30.0.tgz}
     name: date-fns
@@ -6420,12 +6568,6 @@ packages:
     dependencies:
       ms: registry.npmmirror.com/ms@2.1.2
 
-  registry.npmmirror.com/decimal.js@10.4.3:
-    resolution: {integrity: sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/decimal.js/-/decimal.js-10.4.3.tgz}
-    name: decimal.js
-    version: 10.4.3
-    dev: false
-
   registry.npmmirror.com/decode-named-character-reference@1.0.2:
     resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz}
     name: decode-named-character-reference
@@ -6598,6 +6740,16 @@ packages:
       entities: registry.npmmirror.com/entities@2.2.0
     dev: true
 
+  registry.npmmirror.com/dom-serializer@2.0.0:
+    resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dom-serializer/-/dom-serializer-2.0.0.tgz}
+    name: dom-serializer
+    version: 2.0.0
+    dependencies:
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      entities: registry.npmmirror.com/entities@4.5.0
+    dev: false
+
   registry.npmmirror.com/domain-browser@4.23.0:
     resolution: {integrity: sha512-ArzcM/II1wCCujdCNyQjXrAFwS4mrLh4C7DZWlaI8mdh7h3BfKdNd3bKXITfl2PT9FtfQqaGvhi1vPRQPimjGA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domain-browser/-/domain-browser-4.23.0.tgz}
     name: domain-browser
@@ -6609,16 +6761,6 @@ packages:
     resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz}
     name: domelementtype
     version: 2.3.0
-    dev: true
-
-  registry.npmmirror.com/domexception@4.0.0:
-    resolution: {integrity: sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domexception/-/domexception-4.0.0.tgz}
-    name: domexception
-    version: 4.0.0
-    engines: {node: '>=12'}
-    dependencies:
-      webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
-    dev: false
 
   registry.npmmirror.com/domhandler@4.3.1:
     resolution: {integrity: sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domhandler/-/domhandler-4.3.1.tgz}
@@ -6629,6 +6771,15 @@ packages:
       domelementtype: registry.npmmirror.com/domelementtype@2.3.0
     dev: true
 
+  registry.npmmirror.com/domhandler@5.0.3:
+    resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domhandler/-/domhandler-5.0.3.tgz}
+    name: domhandler
+    version: 5.0.3
+    engines: {node: '>= 4'}
+    dependencies:
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+    dev: false
+
   registry.npmmirror.com/dompurify@3.0.6:
     resolution: {integrity: sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/dompurify/-/dompurify-3.0.6.tgz}
     name: dompurify
@@ -6645,6 +6796,16 @@ packages:
       domhandler: registry.npmmirror.com/domhandler@4.3.1
     dev: true
 
+  registry.npmmirror.com/domutils@3.1.0:
+    resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/domutils/-/domutils-3.1.0.tgz}
+    name: domutils
+    version: 3.1.0
+    dependencies:
+      dom-serializer: registry.npmmirror.com/dom-serializer@2.0.0
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+    dev: false
+
   registry.npmmirror.com/downloadjs@1.4.7:
     resolution: {integrity: sha512-LN1gO7+u9xjU5oEScGFKvXhYf7Y/empUIIEAGBs1LzUq/rg5duiDrkuH5A2lQGd5jfMOb9X9usDa2oVXwJ0U/Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/downloadjs/-/downloadjs-1.4.7.tgz}
     name: downloadjs
@@ -8114,6 +8275,13 @@ packages:
       space-separated-tokens: registry.npmmirror.com/space-separated-tokens@2.0.2
     dev: false
 
+  registry.npmmirror.com/he@1.2.0:
+    resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/he/-/he-1.2.0.tgz}
+    name: he
+    version: 1.2.0
+    hasBin: true
+    dev: false
+
   registry.npmmirror.com/heap@0.2.7:
     resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/heap/-/heap-0.2.7.tgz}
     name: heap
@@ -8150,15 +8318,6 @@ packages:
     dependencies:
       react-is: registry.npmmirror.com/react-is@16.13.1
 
-  registry.npmmirror.com/html-encoding-sniffer@3.0.0:
-    resolution: {integrity: sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz}
-    name: html-encoding-sniffer
-    version: 3.0.0
-    engines: {node: '>=12'}
-    dependencies:
-      whatwg-encoding: registry.npmmirror.com/whatwg-encoding@2.0.0
-    dev: false
-
   registry.npmmirror.com/html-parse-stringify@3.0.1:
     resolution: {integrity: sha512-KknJ50kTInJ7qIScF3jeaFRpMpE8/lfiTdzf/twXyPBLAGrLRTmkz3AdTnKeh40X8k9L2fdYwEp/42WGXIRGcg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/html-parse-stringify/-/html-parse-stringify-3.0.1.tgz}
     name: html-parse-stringify
@@ -8166,17 +8325,15 @@ packages:
     dependencies:
       void-elements: registry.npmmirror.com/void-elements@3.1.0
 
-  registry.npmmirror.com/http-proxy-agent@5.0.0:
-    resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz}
-    name: http-proxy-agent
-    version: 5.0.0
-    engines: {node: '>= 6'}
+  registry.npmmirror.com/htmlparser2@8.0.2:
+    resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/htmlparser2/-/htmlparser2-8.0.2.tgz}
+    name: htmlparser2
+    version: 8.0.2
     dependencies:
-      '@tootallnate/once': registry.npmmirror.com/@tootallnate/once@2.0.0
-      agent-base: registry.npmmirror.com/agent-base@6.0.2
-      debug: registry.npmmirror.com/debug@4.3.4
-    transitivePeerDependencies:
-      - supports-color
+      domelementtype: registry.npmmirror.com/domelementtype@2.3.0
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      domutils: registry.npmmirror.com/domutils@3.1.0
+      entities: registry.npmmirror.com/entities@4.5.0
     dev: false
 
   registry.npmmirror.com/https-browserify@1.0.0:
@@ -8185,18 +8342,6 @@ packages:
     version: 1.0.0
     dev: true
 
-  registry.npmmirror.com/https-proxy-agent@5.0.1:
-    resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz}
-    name: https-proxy-agent
-    version: 5.0.1
-    engines: {node: '>= 6'}
-    dependencies:
-      agent-base: registry.npmmirror.com/agent-base@6.0.2
-      debug: registry.npmmirror.com/debug@4.3.4
-    transitivePeerDependencies:
-      - supports-color
-    dev: false
-
   registry.npmmirror.com/human-signals@4.3.1:
     resolution: {integrity: sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/human-signals/-/human-signals-4.3.1.tgz}
     name: human-signals
@@ -8572,12 +8717,6 @@ packages:
     engines: {node: '>=12'}
     dev: false
 
-  registry.npmmirror.com/is-potential-custom-element-name@1.0.1:
-    resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz}
-    name: is-potential-custom-element-name
-    version: 1.0.1
-    dev: false
-
   registry.npmmirror.com/is-regex@1.1.4:
     resolution: {integrity: sha512-kvRdxDsxZjhzUX07ZnLydzS1TU/TJlTUHHY4YLL87e37oUA49DfkLqgy+VjFocowy29cKvcSiu+kIv728jTTVg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/is-regex/-/is-regex-1.1.4.tgz}
     name: is-regex
@@ -8750,46 +8889,6 @@ packages:
     engines: {node: '>=0.1.90'}
     dev: false
 
-  registry.npmmirror.com/jsdom@22.1.0:
-    resolution: {integrity: sha512-/9AVW7xNbsBv6GfWho4TTNjEo9fe6Zhf9O7s0Fhhr3u+awPwAJMKwAMXnkk5vBxflqLW9hTHX/0cs+P3gW+cQw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsdom/-/jsdom-22.1.0.tgz}
-    name: jsdom
-    version: 22.1.0
-    engines: {node: '>=16'}
-    peerDependencies:
-      canvas: ^2.5.0
-    peerDependenciesMeta:
-      canvas:
-        optional: true
-    dependencies:
-      abab: registry.npmmirror.com/abab@2.0.6
-      cssstyle: registry.npmmirror.com/cssstyle@3.0.0
-      data-urls: registry.npmmirror.com/data-urls@4.0.0
-      decimal.js: registry.npmmirror.com/decimal.js@10.4.3
-      domexception: registry.npmmirror.com/domexception@4.0.0
-      form-data: registry.npmmirror.com/form-data@4.0.0
-      html-encoding-sniffer: registry.npmmirror.com/html-encoding-sniffer@3.0.0
-      http-proxy-agent: registry.npmmirror.com/http-proxy-agent@5.0.0
-      https-proxy-agent: registry.npmmirror.com/https-proxy-agent@5.0.1
-      is-potential-custom-element-name: registry.npmmirror.com/is-potential-custom-element-name@1.0.1
-      nwsapi: registry.npmmirror.com/nwsapi@2.2.7
-      parse5: registry.npmmirror.com/parse5@7.1.2
-      rrweb-cssom: registry.npmmirror.com/rrweb-cssom@0.6.0
-      saxes: registry.npmmirror.com/saxes@6.0.0
-      symbol-tree: registry.npmmirror.com/symbol-tree@3.2.4
-      tough-cookie: registry.npmmirror.com/tough-cookie@4.1.3
-      w3c-xmlserializer: registry.npmmirror.com/w3c-xmlserializer@4.0.0
-      webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
-      whatwg-encoding: registry.npmmirror.com/whatwg-encoding@2.0.0
-      whatwg-mimetype: registry.npmmirror.com/whatwg-mimetype@3.0.0
-      whatwg-url: registry.npmmirror.com/whatwg-url@12.0.1
-      ws: registry.npmmirror.com/ws@8.14.2
-      xml-name-validator: registry.npmmirror.com/xml-name-validator@4.0.0
-    transitivePeerDependencies:
-      - bufferutil
-      - supports-color
-      - utf-8-validate
-    dev: false
-
   registry.npmmirror.com/jsesc@0.5.0:
     resolution: {integrity: sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/jsesc/-/jsesc-0.5.0.tgz}
     name: jsesc
@@ -10172,6 +10271,24 @@ packages:
       whatwg-url: registry.npmmirror.com/whatwg-url@5.0.0
     dev: false
 
+  registry.npmmirror.com/node-html-markdown@1.3.0:
+    resolution: {integrity: sha512-OeFi3QwC/cPjvVKZ114tzzu+YoR+v9UXW5RwSXGUqGb0qCl0DvP406tzdL7SFn8pZrMyzXoisfG2zcuF9+zw4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-html-markdown/-/node-html-markdown-1.3.0.tgz}
+    name: node-html-markdown
+    version: 1.3.0
+    engines: {node: '>=10.0.0'}
+    dependencies:
+      node-html-parser: registry.npmmirror.com/node-html-parser@6.1.11
+    dev: false
+
+  registry.npmmirror.com/node-html-parser@6.1.11:
+    resolution: {integrity: sha512-FAgwwZ6h0DSDWxfD0Iq1tsDcBCxdJB1nXpLPPxX8YyVWzbfCjKWEzaynF4gZZ/8hziUmp7ZSaKylcn0iKhufUQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-html-parser/-/node-html-parser-6.1.11.tgz}
+    name: node-html-parser
+    version: 6.1.11
+    dependencies:
+      css-select: registry.npmmirror.com/css-select@5.1.0
+      he: registry.npmmirror.com/he@1.2.0
+    dev: false
+
   registry.npmmirror.com/node-releases@2.0.13:
     resolution: {integrity: sha512-uYr7J37ae/ORWdZeQ1xxMJe3NtdmqMC/JZK+geofDrkLUApKRHPd18/TxtBOJ4A0/+uUIliorNrfYV6s1b02eQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-releases/-/node-releases-2.0.13.tgz}
     name: node-releases
@@ -10245,13 +10362,6 @@ packages:
     version: 2.1.1
     dependencies:
       boolbase: registry.npmmirror.com/boolbase@1.0.0
-    dev: true
-
-  registry.npmmirror.com/nwsapi@2.2.7:
-    resolution: {integrity: sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/nwsapi/-/nwsapi-2.2.7.tgz}
-    name: nwsapi
-    version: 2.2.7
-    dev: false
 
   registry.npmmirror.com/object-assign@4.1.1:
     resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz}
@@ -10532,12 +10642,22 @@ packages:
       json-parse-even-better-errors: registry.npmmirror.com/json-parse-even-better-errors@2.3.1
       lines-and-columns: registry.npmmirror.com/lines-and-columns@1.2.4
 
+  registry.npmmirror.com/parse5-htmlparser2-tree-adapter@7.0.0:
+    resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz}
+    name: parse5-htmlparser2-tree-adapter
+    version: 7.0.0
+    dependencies:
+      domhandler: registry.npmmirror.com/domhandler@5.0.3
+      parse5: registry.npmmirror.com/parse5@7.1.2
+    dev: false
+
   registry.npmmirror.com/parse5@7.1.2:
     resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/parse5/-/parse5-7.1.2.tgz}
     name: parse5
     version: 7.1.2
     dependencies:
       entities: registry.npmmirror.com/entities@4.5.0
+    dev: false
 
   registry.npmmirror.com/path-browserify@1.0.1:
     resolution: {integrity: sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/path-browserify/-/path-browserify-1.0.1.tgz}
@@ -10889,12 +11009,6 @@ packages:
     version: 1.1.0
     dev: false
 
-  registry.npmmirror.com/psl@1.9.0:
-    resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/psl/-/psl-1.9.0.tgz}
-    name: psl
-    version: 1.9.0
-    dev: false
-
   registry.npmmirror.com/public-encrypt@4.0.3:
     resolution: {integrity: sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/public-encrypt/-/public-encrypt-4.0.3.tgz}
     name: public-encrypt
@@ -10935,12 +11049,6 @@ packages:
     engines: {node: '>=0.4.x'}
     dev: true
 
-  registry.npmmirror.com/querystringify@2.2.0:
-    resolution: {integrity: sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/querystringify/-/querystringify-2.2.0.tgz}
-    name: querystringify
-    version: 2.2.0
-    dev: false
-
   registry.npmmirror.com/queue-microtask@1.2.3:
     resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/queue-microtask/-/queue-microtask-1.2.3.tgz}
     name: queue-microtask
@@ -11457,12 +11565,6 @@ packages:
     engines: {node: '>=4'}
     dev: false
 
-  registry.npmmirror.com/requires-port@1.0.0:
-    resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/requires-port/-/requires-port-1.0.0.tgz}
-    name: requires-port
-    version: 1.0.0
-    dev: false
-
   registry.npmmirror.com/resolve-from@4.0.0:
     resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/resolve-from/-/resolve-from-4.0.0.tgz}
     name: resolve-from
@@ -11553,12 +11655,6 @@ packages:
       fsevents: registry.npmmirror.com/fsevents@2.3.3
     dev: true
 
-  registry.npmmirror.com/rrweb-cssom@0.6.0:
-    resolution: {integrity: sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz}
-    name: rrweb-cssom
-    version: 0.6.0
-    dev: false
-
   registry.npmmirror.com/run-parallel@1.2.0:
     resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/run-parallel/-/run-parallel-1.2.0.tgz}
     name: run-parallel
@@ -11649,15 +11745,6 @@ packages:
       immutable: registry.npmmirror.com/immutable@4.3.4
       source-map-js: registry.npmmirror.com/source-map-js@1.0.2
 
-  registry.npmmirror.com/saxes@6.0.0:
-    resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/saxes/-/saxes-6.0.0.tgz}
-    name: saxes
-    version: 6.0.0
-    engines: {node: '>=v12.22.7'}
-    dependencies:
-      xmlchars: registry.npmmirror.com/xmlchars@2.2.0
-    dev: false
-
   registry.npmmirror.com/scheduler@0.23.0:
     resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/scheduler/-/scheduler-0.23.0.tgz}
     name: scheduler
@@ -12100,12 +12187,6 @@ packages:
       stable: registry.npmmirror.com/stable@0.1.8
     dev: true
 
-  registry.npmmirror.com/symbol-tree@3.2.4:
-    resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/symbol-tree/-/symbol-tree-3.2.4.tgz}
-    name: symbol-tree
-    version: 3.2.4
-    dev: false
-
   registry.npmmirror.com/tapable@2.2.1:
     resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tapable/-/tapable-2.2.1.tgz}
     name: tapable
@@ -12180,18 +12261,6 @@ packages:
     version: 1.0.6
     dev: false
 
-  registry.npmmirror.com/tough-cookie@4.1.3:
-    resolution: {integrity: sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tough-cookie/-/tough-cookie-4.1.3.tgz}
-    name: tough-cookie
-    version: 4.1.3
-    engines: {node: '>=6'}
-    dependencies:
-      psl: registry.npmmirror.com/psl@1.9.0
-      punycode: registry.npmmirror.com/punycode@2.3.0
-      universalify: registry.npmmirror.com/universalify@0.2.0
-      url-parse: registry.npmmirror.com/url-parse@1.5.10
-    dev: false
-
   registry.npmmirror.com/tr46@0.0.3:
     resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-0.0.3.tgz}
     name: tr46
@@ -12207,15 +12276,6 @@ packages:
       punycode: registry.npmmirror.com/punycode@2.3.0
     dev: false
 
-  registry.npmmirror.com/tr46@4.1.1:
-    resolution: {integrity: sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/tr46/-/tr46-4.1.1.tgz}
-    name: tr46
-    version: 4.1.1
-    engines: {node: '>=14'}
-    dependencies:
-      punycode: registry.npmmirror.com/punycode@2.3.0
-    dev: false
-
   registry.npmmirror.com/trim-lines@3.0.1:
     resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/trim-lines/-/trim-lines-3.0.1.tgz}
     name: trim-lines
@@ -12615,13 +12675,6 @@ packages:
       unist-util-visit-parents: registry.npmmirror.com/unist-util-visit-parents@5.1.3
     dev: false
 
-  registry.npmmirror.com/universalify@0.2.0:
-    resolution: {integrity: sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/universalify/-/universalify-0.2.0.tgz}
-    name: universalify
-    version: 0.2.0
-    engines: {node: '>= 4.0.0'}
-    dev: false
-
   registry.npmmirror.com/update-browserslist-db@1.0.13(browserslist@4.22.1):
     resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz}
     id: registry.npmmirror.com/update-browserslist-db/1.0.13
@@ -12643,15 +12696,6 @@ packages:
       punycode: registry.npmmirror.com/punycode@2.3.0
     dev: true
 
-  registry.npmmirror.com/url-parse@1.5.10:
-    resolution: {integrity: sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/url-parse/-/url-parse-1.5.10.tgz}
-    name: url-parse
-    version: 1.5.10
-    dependencies:
-      querystringify: registry.npmmirror.com/querystringify@2.2.0
-      requires-port: registry.npmmirror.com/requires-port@1.0.0
-    dev: false
-
   registry.npmmirror.com/url@0.11.3:
     resolution: {integrity: sha512-6hxOLGfZASQK/cijlZnZJTq8OXAkt/3YGfQX45vvMYXpZoo8NdWZcY73K108Jf759lS1Bv/8wXnHDTSz17dSRw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/url/-/url-0.11.3.tgz}
     name: url
@@ -12902,15 +12946,6 @@ packages:
     version: 3.1.0
     engines: {node: '>=0.10.0'}
 
-  registry.npmmirror.com/w3c-xmlserializer@4.0.0:
-    resolution: {integrity: sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz}
-    name: w3c-xmlserializer
-    version: 4.0.0
-    engines: {node: '>=14'}
-    dependencies:
-      xml-name-validator: registry.npmmirror.com/xml-name-validator@4.0.0
-    dev: false
-
   registry.npmmirror.com/watchpack@2.4.0:
     resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/watchpack/-/watchpack-2.4.0.tgz}
     name: watchpack
@@ -12959,22 +12994,6 @@ packages:
     engines: {node: '>=12'}
     dev: false
 
-  registry.npmmirror.com/whatwg-encoding@2.0.0:
-    resolution: {integrity: sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz}
-    name: whatwg-encoding
-    version: 2.0.0
-    engines: {node: '>=12'}
-    dependencies:
-      iconv-lite: registry.npmmirror.com/iconv-lite@0.6.3
-    dev: false
-
-  registry.npmmirror.com/whatwg-mimetype@3.0.0:
-    resolution: {integrity: sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz}
-    name: whatwg-mimetype
-    version: 3.0.0
-    engines: {node: '>=12'}
-    dev: false
-
   registry.npmmirror.com/whatwg-url@11.0.0:
     resolution: {integrity: sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-11.0.0.tgz}
     name: whatwg-url
@@ -12985,16 +13004,6 @@ packages:
       webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
     dev: false
 
-  registry.npmmirror.com/whatwg-url@12.0.1:
-    resolution: {integrity: sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-12.0.1.tgz}
-    name: whatwg-url
-    version: 12.0.1
-    engines: {node: '>=14'}
-    dependencies:
-      tr46: registry.npmmirror.com/tr46@4.1.1
-      webidl-conversions: registry.npmmirror.com/webidl-conversions@7.0.0
-    dev: false
-
   registry.npmmirror.com/whatwg-url@5.0.0:
     resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/whatwg-url/-/whatwg-url-5.0.0.tgz}
     name: whatwg-url
@@ -13137,28 +13146,6 @@ packages:
     name: wrappy
     version: 1.0.2
 
-  registry.npmmirror.com/ws@8.14.2:
-    resolution: {integrity: sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/ws/-/ws-8.14.2.tgz}
-    name: ws
-    version: 8.14.2
-    engines: {node: '>=10.0.0'}
-    peerDependencies:
-      bufferutil: ^4.0.1
-      utf-8-validate: '>=5.0.2'
-    peerDependenciesMeta:
-      bufferutil:
-        optional: true
-      utf-8-validate:
-        optional: true
-    dev: false
-
-  registry.npmmirror.com/xml-name-validator@4.0.0:
-    resolution: {integrity: sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz}
-    name: xml-name-validator
-    version: 4.0.0
-    engines: {node: '>=12'}
-    dev: false
-
   registry.npmmirror.com/xmlbuilder@10.1.1:
     resolution: {integrity: sha512-OyzrcFLL/nb6fMGHbiRDuPup9ljBycsdCypwuyg5AAHvyWzGfChJpCXMG88AGTIMFhGZ9RccFN1e6lhg3hkwKg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlbuilder/-/xmlbuilder-10.1.1.tgz}
     name: xmlbuilder
@@ -13166,12 +13153,6 @@ packages:
     engines: {node: '>=4.0'}
     dev: false
 
-  registry.npmmirror.com/xmlchars@2.2.0:
-    resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xmlchars/-/xmlchars-2.2.0.tgz}
-    name: xmlchars
-    version: 2.2.0
-    dev: false
-
   registry.npmmirror.com/xtend@4.0.2:
     resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/xtend/-/xtend-4.0.2.tgz}
     name: xtend

projects/app/data/config.json

@@ -113,6 +113,7 @@
       "maxToken": 3000
     }
   ],
+  "ReRankModels": [],
   "AudioSpeechModels": [
     {
       "model": "tts-1",

projects/app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app",
-  "version": "4.6.1",
+  "version": "4.6.3",
   "private": false,
   "scripts": {
     "dev": "next dev",
@@ -19,7 +19,7 @@
     "@fastgpt/global": "workspace:*",
     "@fastgpt/service": "workspace:*",
     "@fastgpt/web": "workspace:*",
-    "@mozilla/readability": "^0.4.4",
+    "@node-rs/jieba": "^1.7.2",
     "@tanstack/react-query": "^4.24.10",
     "@types/nprogress": "^0.2.0",
     "axios": "^1.5.1",
@@ -34,10 +34,9 @@
     "i18next": "^22.5.1",
     "immer": "^9.0.19",
     "jschardet": "^3.0.0",
-    "jsdom": "^22.1.0",
     "jsonwebtoken": "^9.0.2",
     "lodash": "^4.17.21",
-    "mammoth": "^1.5.1",
+    "mammoth": "^1.6.0",
     "mermaid": "^10.2.3",
     "multer": "1.4.5-lts.1",
     "nanoid": "^4.0.1",
@@ -66,7 +65,6 @@
     "@types/downloadjs": "^1.4.3",
     "@types/formidable": "^2.0.5",
     "@types/js-cookie": "^3.0.3",
-    "@types/jsdom": "^21.1.1",
     "@types/jsonwebtoken": "^9.0.3",
     "@types/lodash": "^4.14.191",
     "@types/multer": "^1.4.10",

projects/app/public/docs/versionIntro.md

@@ -1,12 +1,11 @@
-### Fast GPT V4.6
+### Fast GPT V4.6.2
 
-1. 新增 - 团队空间
-2. 新增 - 多路向量（多个向量映射一组数据）
-3. 新增 - tts语音
-4. 线上环境新增 - ReRank向量召回，提高召回精度
-5. 优化 - 知识库导出，可直接触发流下载，无需等待转圈圈
-6. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
-7. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
-8. [使用文档](https://doc.fastgpt.in/docs/intro/)
-9. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
-10. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)
+1. 商业版新增 - web站点同步
+2. 新增 - 集合元数据记录
+3. 优化 - url 读取内容
+4. 优化 - 流读取文件，防止内存溢出
+5. [知识库结构详解](https://doc.fastgpt.in/docs/use-cases/datasetengine/)
+6. [知识库提示词详解](https://doc.fastgpt.in/docs/use-cases/ai_settings/#引用模板--引用提示词)
+7. [使用文档](https://doc.fastgpt.in/docs/intro/)
+8. [点击查看高级编排介绍文档](https://doc.fastgpt.in/docs/workflow)
+9. [点击查看商业版](https://doc.fastgpt.in/docs/commercial/)

projects/app/public/imgs/modal/badAnswer.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700745147176" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="25531" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M551.706122 909.061224c-25.6 0-49.632653-8.359184-65.306122-24.555102-25.6-26.122449-36.571429-70.530612-47.542857-117.028571-6.269388-26.122449-13.583673-55.902041-21.420408-68.963265-20.897959-35.526531-65.828571-54.334694-78.367347-59.036735H266.971429c-25.6 0-47.020408-20.897959-47.020409-47.020408V215.771429c0-25.6 20.897959-47.020408 47.020409-47.020409h67.918367C364.146939 153.6 447.738776 114.938776 543.346939 114.938776h181.812245c29.779592 0 92.473469 10.971429 117.55102 53.289795 23.510204 39.183673 63.216327 344.293878 63.216327 370.416327 0 56.946939-53.812245 101.355102-100.310204 101.355102h-119.640817c-4.179592 0-6.791837 2.089796-7.836734 3.657143-1.044898 1.567347-2.612245 4.179592-2.089796 8.359184 4.179592 20.37551 13.583673 73.665306 6.791836 130.612244-7.314286 57.991837-41.795918 102.922449-92.473469 120.163266-12.016327 4.179592-25.6 6.269388-38.661225 6.269387z" fill="#16C4AF" p-id="25532"></path><path d="M132.179592 623.281633c-10.971429 0-19.853061-8.881633-19.853061-19.853062V196.440816c0-10.971429 8.881633-19.853061 19.853061-19.853061s19.853061 8.881633 19.853061 19.853061v406.987755c0 10.971429-8.881633 19.853061-19.853061 19.853062z" fill="#16C4AF" p-id="25533"></path></svg>

projects/app/public/imgs/modal/bill.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700746245397" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="43680" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M0 0m178.086957 0l667.826086 0q178.086957 0 178.086957 178.086957l0 667.826086q0 178.086957-178.086957 178.086957l-667.826086 0q-178.086957 0-178.086957-178.086957l0-667.826086q0-178.086957 178.086957-178.086957Z" fill="#8973E6" p-id="43681"></path><path d="M570.813217 258.782609a77.534609 77.534609 0 0 1 77.534609 77.534608v350.987131a77.913043 77.913043 0 0 0 74.017391 77.824l3.895653 0.089043H316.838957a77.534609 77.534609 0 0 1-77.534609-77.534608V336.317217a77.534609 77.534609 0 0 1 77.534609-77.534608h253.996521zM456.43687 589.913043h-122.61287a16.606609 16.606609 0 0 0-16.606609 16.606609v5.743305c0 9.171478 7.43513 16.606609 16.606609 16.606608h122.61287a16.606609 16.606609 0 0 0 16.606608-16.606608v-5.743305a16.606609 16.606609 0 0 0-16.606608-16.606609z m77.913043-116.869565h-200.525913a16.606609 16.606609 0 0 0-16.606609 16.606609v5.743304c0 9.171478 7.43513 16.606609 16.606609 16.606609h200.525913a16.606609 16.606609 0 0 0 16.606609-16.606609v-5.743304a16.606609 16.606609 0 0 0-16.606609-16.606609z m0-116.869565h-200.525913a16.606609 16.606609 0 0 0-16.606609 16.606609v5.743304c0 9.171478 7.43513 16.606609 16.606609 16.606609h200.525913a16.606609 16.606609 0 0 0 16.606609-16.606609v-5.743304a16.606609 16.606609 0 0 0-16.606609-16.606609z" fill="#FFFFFF" p-id="43682"></path><path d="M628.869565 570.434783h155.826087v116.869565a77.913043 77.913043 0 0 1-155.826087 0v-116.869565z" fill="#FFFFFF" opacity=".6" p-id="43683"></path></svg>

projects/app/public/imgs/modal/chatHistory.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700627014976" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="9004" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M749.57747 338.71354a28.03513 28.03513 0 0 1-1.921508-56.00726l167.832777-11.875555-5.355025-146.507178a28.03513 28.03513 0 0 1 56.038759-2.047509l5.607026 153.43721a48.730726 48.730726 0 0 1-45.45471 50.589234l-174.762809 12.348057c-0.630003 0.063-1.291506 0.063-1.98451 0.063001z m166.16327-60.952783z" fill="#2E3138" p-id="9005"></path><path d="M585.461711 1004.878624a484.125741 484.125741 0 0 0-190.544383-929.254302H285.13932l-94.815439 74.466344-152.775707 469.761675L395.35833 996.153084s131.607609 33.799656 190.103381 8.72554z" fill="#56E5BE" p-id="9006"></path><path d="M769.202061 559.781563a484.220242 484.220242 0 0 1-290.179343 443.648054c-59.346275 25.98762-339.06757-61.173283-369.559711-198.450919-63.630295-286.651327-5.071523-729.133876 175.644813-729.133875a484.031241 484.031241 0 0 1 484.094241 483.93674z" fill="#50DDB8" p-id="9007"></path><path d="M279.658295 87.342376a484.220242 484.220242 0 0 1 103.950481 908.810708l-233.101079-144.900671-112.802022-242.551123 8.851541-229.258062 107.100495-210.389474z" fill="#42D3AD" p-id="9008"></path><path d="M512.727874 1023.999212a512.003371 512.003371 0 0 1-426.921477-227.998055C-70.780828 561.041569-6.993032 242.480594 227.998056 85.798869S781.424118-6.87456 938.105843 228.085028a28.06663 28.06663 0 0 1-46.683216 31.122144 455.964611 455.964611 0 0 0-632.333927-126.630587C49.864731 271.996231-6.961532 555.655044 132.521114 764.910513s423.046959 266.050232 632.333927 126.599086a458.925625 458.925625 0 0 0 198.954921-441.821045 28.04458 28.04458 0 1 1 55.566258-7.623036 515.342386 515.342386 0 0 1-223.462035 496.127297 509.451859 509.451859 0 0 1-283.186311 85.806397z" fill="#2E3138" p-id="9009"></path><path d="M729.543378 657.558016h-2.236511l-186.354862-14.553067a47.029718 47.029718 0 0 1-43.0292-46.746217v-211.77548a28.03513 28.03513 0 0 1 56.07026 0v203.17594l177.818323 13.891565a28.03513 28.03513 0 0 1-2.14201 56.007259z" fill="#2E3138" p-id="9010"></path></svg>

projects/app/public/imgs/modal/concat.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700745458924" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="30191" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M0 512a512 512 0 1 0 1024 0 512 512 0 1 0-1024 0Z" fill="#EFF2FF" p-id="30192"></path><path d="M682.666667 682.666667c9.898667 9.841778-9.841778-9.841778 0 0zM338.488889 685.511111l5.688889-5.688889c-1.080889 0.682667-2.161778 2.104889-2.844445 2.844445a10.752 10.752 0 0 0-2.844444 2.844444z" fill="#000000" p-id="30193"></path><path d="M784.896 740.693333c23.324444-40.504889 10.410667-71.68-30.947556-102.798222-48.696889-36.579556-96.199111-61.212444-130.389333-21.788444 0 0-36.295111 43.064889-143.075555-57.685334C356.352 440.490667 408.462222 398.677333 408.462222 398.677333c43.178667-43.235556 15.701333-75.548444-20.48-124.416-36.238222-48.924444-72.817778-64.341333-125.326222-22.300444-101.262222 81.009778 41.528889 271.189333 113.607111 345.144889 0 0 109.738667 113.095111 178.801778 150.755555l36.920889 20.593778c52.906667 27.079111 112.469333 39.480889 154.339555 14.336 0 0 20.195556-10.353778 38.570667-42.097778z" fill="#4D4DEE" p-id="30194"></path></svg>

projects/app/public/imgs/modal/edit.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700634007483" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="19559" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M65.361 856H967v160H65.361zM65.531 805.062l2.295-188.972L676.402 7.515 863.078 194.19 254.503 802.766 65.53 805.062z m50.726-169.52l-1.46 120.254 120.254-1.46L116.257 635.54z m507.147-507.147L742.198 247.19l52.163-52.163L675.567 76.232l-52.163 52.163z" fill="#1AA5FF" p-id="19560"></path></svg>

projects/app/public/imgs/modal/folder.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700746526624" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="53489" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M300.053794 0h423.892412c200.032053 0 300.053794 100.021741 300.053794 300.053794v423.892412c0 200.032053-100.021741 300.053794-300.053794 300.053794H300.053794C100.021741 1023.988572 0 923.966831 0 723.934778v-423.880984C0 100.021741 100.021741 0 300.053794 0z" fill="#598BFC" opacity=".15" p-id="53490"></path><path d="M709.900648 336.864812h-191.883572c-7.017065 0-13.542706-3.645674-17.211237-9.62275l-43.725226-41.828105a34.641899 34.641899 0 0 0-29.508242-16.479816h-110.695908c-43.725226 0-79.164831 35.416748-79.16483 79.09626v310.876531c0 42.136673 34.182476 76.296291 76.364862 76.30772h395.835582c42.170958 0 76.364862-34.171047 76.364862-76.30772V413.172532c0-42.136673-34.193904-76.30772-76.376291-76.30772z" fill="#598BFC" p-id="53491"></path><path d="M335.436256 437.800828h353.116059c13.336994 0 20.011205 6.857066 20.011205 20.571199s-6.674211 20.571199-20.011205 20.571199H335.436256c-13.336994 0-20.011205-6.857066-20.011205-20.571199s6.674211-20.571199 20.011205-20.571199z" fill="#FFFFFF" p-id="53492"></path></svg>

projects/app/public/imgs/modal/import.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700746705101" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="61761" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M906.5472 756.8384H632.9344c-23.04 0-43.2128-20.1728-43.2128-43.2128s20.1728-43.2128 43.2128-43.2128h273.6128c23.04 0 43.2128 20.1728 43.2128 43.2128s-20.1728 43.2128-43.2128 43.2128z" fill="#409EFF" p-id="61762"></path><path d="M631.296 756.8384c-11.5712 0-23.04-2.8672-28.7744-11.5712-17.3056-17.3056-17.3056-43.2128 0-60.5184L732.16 555.2128c17.3056-17.3056 43.2128-17.3056 60.5184 0s17.3056 43.2128 0 60.5184L663.04 745.3696c-11.5712 8.6016-23.04 11.4688-31.744 11.4688z" fill="#409EFF" p-id="61763"></path><path d="M761.4464 886.3744c-11.5712 0-23.04-2.8672-28.7744-11.5712L603.0336 745.3696c-17.3056-17.3056-17.3056-43.2128 0-60.5184s43.2128-17.3056 60.5184 0l129.6384 129.536c17.3056 17.3056 17.3056 43.2128 0 60.5184-8.704 5.7344-20.2752 11.4688-31.744 11.4688z" fill="#409EFF" p-id="61764"></path><path d="M851.8656 932.4544a126.65856 126.65856 0 0 1-89.2928 37.4784c-34.6112 0-63.3856-11.5712-89.2928-37.4784L546.5088 802.9184c-17.3056-17.3056-28.7744-40.3456-34.6112-63.3856-2.8672-8.6016-2.8672-17.3056-2.8672-25.9072s0-17.3056 2.8672-25.9072c5.7344-23.04 17.3056-46.08 34.6112-63.3856L676.0448 494.592c48.4352-49.3568 127.6928-49.9712 177.0496-1.536l1.536 1.536c17.6128 17.6128 52.0192 62.1568 67.2768 82.1248 1.4336 1.8432 4.096 2.2528 5.9392 0.8192 1.024-0.8192 1.6384-2.048 1.6384-3.3792V280.064c0-65.6384-53.76-119.3984-119.3984-119.3984H314.5728c-65.6384 0-119.3984 53.76-119.3984 119.3984v573.3376c0 65.6384 53.76 119.3984 119.3984 119.3984h492.6464c65.6384 0 119.3984-53.76 119.3984-119.3984 0-2.3552-1.8432-4.1984-4.1984-4.1984a3.9936 3.9936 0 0 0-3.3792 1.7408c-14.7456 20.1728-47.616 64.1024-67.1744 81.5104z" fill="#409EFF" p-id="61765"></path><path d="M131.7888 823.0912V212.48c0-66.2528 54.6816-120.9344 120.9344-120.9344h524.1856C756.8384 68.5056 725.0944 51.2 690.5856 51.2H189.44C126.0544 51.2 74.24 103.0144 74.24 166.4v584.6016c0 43.2128 23.04 80.5888 60.5184 100.7616-2.9696-11.4688-2.9696-20.0704-2.9696-28.672z" fill="#409EFF" p-id="61766"></path></svg>

projects/app/public/imgs/modal/key.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700983497588" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="6628" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M698.483573 594.905936A287.506808 287.506808 0 1 1 984.611923 306.020671v1.181535a287.309885 287.309885 0 0 1-286.12835 287.70373z" fill="#FFFFFF" p-id="6629"></path><path d="M698.483573 39.387645A267.814561 267.814561 0 1 1 433.229005 308.777585v-1.575379A267.420716 267.420716 0 0 1 698.483573 39.387645m0-39.384494A307.199055 307.199055 0 1 0 1004.30417 308.580663v-1.378457A306.411365 306.411365 0 0 0 698.680495 0.003151z" fill="#007FB7" p-id="6630"></path><path d="M787.689452 236.310116m-78.768988 0a78.768988 78.768988 0 1 0 157.537977 0 78.768988 78.768988 0 1 0-157.537977 0Z" fill="#D1EBF2" p-id="6631"></path><path d="M787.689452 177.233375a59.076741 59.076741 0 1 1-59.076741 59.076741 59.076741 59.076741 0 0 1 59.076741-59.076741m0-39.384495a98.461236 98.461236 0 1 0 98.461236 98.461236 98.461236 98.461236 0 0 0-98.461236-98.461236z" fill="#007FB7" p-id="6632"></path><path d="M39.384062 974.57246v-113.033499l390.300338-392.466484 162.067194 108.701204-116.381181 58.682896v124.455002l-135.876505 5.316906v131.150366l-127.014993 4.923062-65.772106 99.248925L39.384062 974.57246z" fill="#D1EBF2" p-id="6633"></path><path d="M433.229005 494.475475l120.713474 80.935136-75.421306 38.006037-21.661472 10.830736v118.153482l-98.461235 3.741527-38.793727 2.166148v131.347288l-98.461236 3.741527h-19.692247l-11.224581 16.73841L137.845298 979.101677l-78.768988-19.692247v-89.796647l374.152695-374.152695m-5.119985-50.805998L19.691815 853.268218v136.467272L155.56832 1024l67.938253-102.399685 135.876505-5.316907v-131.150365l135.876505-5.316907v-131.347288L630.151476 580.333673l-203.027068-136.664195z" fill="#007FB7" p-id="6634"></path></svg>

projects/app/public/imgs/modal/move.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700746780241" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="67557" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M768 320v128H576V256h128L512 64 320 256h128v192H256V320L64 512l64 64 128 128V576h192v192H320l192 192 64-64 128-128H576V576h192v128l192-192-192-192z" p-id="67558" fill="#13227a"></path></svg>

projects/app/public/imgs/modal/openai.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700746289796" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="44851" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M939.795692 421.415385a248.832 248.832 0 0 0-21.385846-204.386462 251.667692 251.667692 0 0 0-271.064615-120.753231 248.832 248.832 0 0 0-187.687385-83.672615 251.687385 251.687385 0 0 0-240.088615 174.257231 248.910769 248.910769 0 0 0-166.4 120.713846 251.707077 251.707077 0 0 0 30.956307 295.108923 248.832 248.832 0 0 0 21.385847 204.386461 251.687385 251.687385 0 0 0 271.064615 120.753231 248.832 248.832 0 0 0 187.687385 83.672616 251.687385 251.687385 0 0 0 240.167384-174.355693 248.910769 248.910769 0 0 0 166.4-120.713846A251.707077 251.707077 0 0 0 939.795692 421.415385zM564.342154 946.195692a186.663385 186.663385 0 0 1-119.827692-43.323077c1.516308-0.827077 4.174769-2.284308 5.907692-3.347692l198.892308-114.884923a32.334769 32.334769 0 0 0 16.344615-28.297846V475.943385l84.066461 48.541538a2.993231 2.993231 0 0 1 1.634462 2.304v232.211692a187.431385 187.431385 0 0 1-187.017846 187.195077z m-402.195692-171.776a186.564923 186.564923 0 0 1-22.331077-125.44c1.476923 0.886154 4.056615 2.461538 5.907692 3.524923l198.892308 114.884923a32.374154 32.374154 0 0 0 32.669538 0l242.825846-140.20923v97.083077a3.012923 3.012923 0 0 1-1.201231 2.579692l-201.058461 116.086154a187.392 187.392 0 0 1-255.704615-68.509539z m-52.322462-434.195692a186.505846 186.505846 0 0 1 97.437538-82.077538c0 1.713231-0.098462 4.745846-0.098461 6.852923v229.769846a32.315077 32.315077 0 0 0 16.324923 28.278154L466.313846 663.236923l-84.066461 48.541539a3.012923 3.012923 0 0 1-2.835693 0.256l-201.078154-116.184616a187.392 187.392 0 0 1-68.509538-255.625846z m690.688 160.728615l-242.825846-140.20923 84.066461-48.521847a3.012923 3.012923 0 0 1 2.835693-0.256l201.078154 116.086154a187.234462 187.234462 0 0 1-28.928 337.821539V529.230769a32.295385 32.295385 0 0 0-16.226462-28.278154zM884.184615 375.020308c-1.476923-0.905846-4.056615-2.461538-5.907692-3.524923l-198.892308-114.884923a32.413538 32.413538 0 0 0-32.669538 0l-242.825846 140.20923v-97.083077a3.012923 3.012923 0 0 1 1.201231-2.579692l201.058461-115.987692A187.214769 187.214769 0 0 1 884.184615 375.020308z m-526.00123 173.036307l-84.086154-48.541538a2.993231 2.993231 0 0 1-1.634462-2.304V264.999385a187.214769 187.214769 0 0 1 307.003077-143.753847c-1.516308 0.827077-4.155077 2.284308-5.907692 3.347693l-198.892308 114.884923a32.315077 32.315077 0 0 0-16.344615 28.278154z m45.666461-98.461538L512 387.131077l108.150154 62.424615v124.888616L512 636.868923l-108.150154-62.424615z" fill="#202123" p-id="44852"></path></svg>

projects/app/public/imgs/modal/params.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700626300526" class="icon" viewBox="0 0 1050 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="5944" xmlns:xlink="http://www.w3.org/1999/xlink" width="131.25" height="128"><path d="M877.714286 29.404164H173.484566c-72.53027 0-131.338598 58.808327-131.338598 131.338597V864.972481c0 72.53027 58.808327 131.338598 131.338598 131.338598h704.22972c72.53027 0 131.338598-58.808327 131.338597-131.338598V160.742761C1008.807849 88.212491 949.999521 29.404164 877.714286 29.404164zM857.621441 760.832735h-113.941135c-19.112706 39.695621-59.543431 67.384542-106.590093 67.384542s-87.477387-27.443886-106.590093-67.384542h-318.545107c-27.933955 0-50.722182-22.788227-50.722182-50.722182 0-27.933955 22.788227-50.722182 50.722182-50.722183h318.790141c19.112706-39.695621 59.543431-67.384542 106.590094-67.384541s87.477387 27.443886 106.590093 67.384541H857.621441c27.933955 0 50.722182 22.788227 50.722182 50.722183 0.245035 28.17899-22.543192 50.722182-50.722182 50.722182z m0-394.750897H523.149079c-19.112706 39.695621-59.543431 67.384542-106.590094 67.384542s-87.477387-27.443886-106.590093-67.384542h-98.013879c-27.933955 0-50.722182-22.788227-50.722182-50.722183 0-27.933955 22.788227-50.722182 50.722182-50.722182h98.258914c19.112706-39.695621 59.543431-67.384542 106.590093-67.384542s87.477387 27.443886 106.590093 67.384542H857.621441c27.933955 0 50.722182 22.788227 50.722182 50.722182 0.245035 27.933955-22.543192 50.722182-50.722182 50.722183z" p-id="5945" fill="#1296db"></path></svg>

projects/app/public/imgs/modal/password.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1700746389208" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="49701" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M128 0h768C981.357714 0 1024 42.642286 1024 128v768c0 85.357714-42.642286 128-128 128H128C42.642286 1024 0 981.357714 0 896V128C0 42.642286 42.642286 0 128 0z" fill="#4C84FF" p-id="49702"></path><path d="M704 398.189714h-31.963429v-64.950857c0-89.746286-71.68-162.596571-160.036571-162.596571-88.283429 0-159.963429 72.850286-159.963429 162.596571v64.950857h-32.036571c-35.181714 0-64 29.257143-64 65.097143v282.331429c0 35.766857 28.818286 65.097143 64 65.097143h384c35.181714 0 64-29.257143 64-65.097143v-282.331429a64.731429 64.731429 0 0 0-64-65.097143zM512 682.715429a85.577143 85.577143 0 0 1-85.357714-85.357715A85.577143 85.577143 0 0 1 512 512a85.577143 85.577143 0 0 1 85.357714 85.357714A85.577143 85.577143 0 0 1 512 682.642286z m-95.963429-284.525715v-64.950857c0-53.979429 42.861714-97.572571 95.963429-97.572571 53.101714 0 96.036571 43.52 96.036571 97.572571v64.950857H416.036571z" fill="#FFFFFF" p-id="49703"></path></svg>