Snip test (#3204 )

* fix: index * fix: snapshot error; perf: snapshot diff compare * perf: init simple edit history
refactor: snapshot store to diff (#3155 )
2024-11-21 16:26:43 +08:00 · 2024-11-21 13:12:42 +08:00 · 2024-11-21 12:01:55 +08:00 · 2024-11-19 10:19:21 +08:00 · 2024-11-18 20:57:03 +08:00 · 2024-11-18 18:34:08 +08:00
25 changed files with 602 additions and 216 deletions
--- a/docSite/content/zh-cn/docs/development/configuration.md
+++ b/docSite/content/zh-cn/docs/development/configuration.md
@@ -23,6 +23,7 @@ weight: 708
  "systemEnv": {
    "vectorMaxProcess": 15,
    "qaMaxProcess": 15,
+    "tokenWorkers": 50, // Token 计算线程保持数，会持续占用内存，不能设置太大。
    "pgHNSWEfSearch": 100 // 向量搜索参数。越大，搜索越精确，但是速度越慢。设置为100，有99%+精度。
  },
  "llmModels": [
--- a/docSite/content/zh-cn/docs/development/openapi/chat.md
+++ b/docSite/content/zh-cn/docs/development/openapi/chat.md
@@ -35,9 +35,10 @@ curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
 --header 'Authorization: Bearer fastgpt-xxxxxx' \
 --header 'Content-Type: application/json' \
 --data-raw '{
-    "chatId": "abcd",
+    "chatId": "my_chatId",
    "stream": false,
    "detail": false,
+    "responseChatItemId": "my_responseChatItemId",
    "variables": {
        "uid": "asdfadsfasfd2323",
        "name": "张三"
@@ -104,6 +105,7 @@ curl --location --request POST 'http://localhost:3000/api/v1/chat/completions' \
  - 为 `undefined` 时（不传入），不使用 FastGpt 提供的上下文功能，完全通过传入的 messages 构建上下文。 不会将你的记录存储到数据库中，你也无法在记录汇总中查阅到。
  - 为`非空字符串`时，意味着使用 chatId 进行对话，自动从 FastGpt 数据库取历史记录，并使用 messages 数组最后一个内容作为用户问题。请自行确保 chatId 唯一，长度小于250，通常可以是自己系统的对话框ID。
 - messages: 结构与 [GPT接口](https://platform.openai.com/docs/api-reference/chat/object) chat模式一致。
+- responseChatItemId: string | undefined 。如果传入，则会将该值作为本次对话的响应消息的 ID，FastGPT 会自动将该 ID 存入数据库。请确保，在当前`chatId`下，`responseChatItemId`是唯一的。
 - detail: 是否返回中间值（模块状态，响应的完整结果等），`stream模式`下会通过`event`进行区分，`非stream模式`结果保存在`responseData`中。
 - variables: 模块变量，一个对象，会替换模块中，输入框内容里的`{{key}}`
 {{% /alert %}}
--- a/docSite/content/zh-cn/docs/development/upgrading/4813.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/4813.md
@@ -19,11 +19,11 @@ weight: 811

 ### 3. 添加环境变量

- 给 fastgpt 和 fastgpt-pro 镜像添加环境变量：`FE_DOMAIN=http://xx.com`，值为 fastgpt 前端访问地址，注意后面不要加`/`。如果没加到话，图片识别可能会有问题。
+- 给 fastgpt 和 fastgpt-pro 镜像添加环境变量：`FE_DOMAIN=http://xx.com`，值为 fastgpt 前端访问地址，注意后面不要加`/`。可以自动补齐相对文件地址的前缀。

 ### 4. 调整文件上传编排

-虽然依然兼容旧版的文件上传编排，但是未来两个版本内将会去除兼容代码，请尽快调整编排，以适应最新的文件上传逻辑。尤其是嵌套应用的文件传递，未来将不会自动传递，必须手动指定传递的文件。
+虽然依然兼容旧版的文件上传编排，但是未来两个版本内将会去除兼容代码，请尽快调整编排，以适应最新的文件上传逻辑。尤其是嵌套应用的文件传递，未来将不会自动传递，必须手动指定传递的文件。具体内容可参考: [文件上传变更](/docs/guide/course/fileinput/#4813%E7%89%88%E6%9C%AC%E8%B5%B7%E5%85%B3%E4%BA%8E%E6%96%87%E4%BB%B6%E4%B8%8A%E4%BC%A0%E7%9A%84%E6%9B%B4%E6%96%B0)

 ## 更新说明

--- a/docSite/content/zh-cn/docs/development/upgrading/4814.md
+++ b/docSite/content/zh-cn/docs/development/upgrading/4814.md
@@ -11,3 +11,9 @@ weight: 810

 1. 
 2. 新增 - 工作流支持进入聊天框/点击开始对话后，自动触发一轮对话。
+3. 新增 - 重写 chatContext，对话测试也会有日志，并且刷新后不会丢失对话。
+4. 新增 - 分享链接支持配置是否允许查看原文。
+5. 优化 - 工作流 ui 细节。
+6. 优化 - 应用编辑记录采用 diff 存储，避免浏览器溢出。
+7. 修复 - 分块策略，四级标题会被丢失。 同时新增了五级标题的支持。
+8. 修复 - MongoDB 知识库集合唯一索引。
--- a/docSite/content/zh-cn/docs/use-cases/external-integration/official_account.md
+++ b/docSite/content/zh-cn/docs/use-cases/external-integration/official_account.md
@@ -37,40 +37,47 @@ weight: 506
 私有部署的用户可自行查阅自己的 IP 地址。

 海外版用户（cloud.tryfastgpt.ai)可以填写下面的 IP 白名单：
-如果仍无响应，可输入命令: `nslookup cloud.sealos.io | awk '/^Address: [0-9]/ {print $2}'` 获取最新 IP

 ```
-34.143.240.160
-35.197.149.75
-34.87.173.252
-34.87.20.189
-34.87.44.74
-34.124.189.116
-34.126.163.205
-35.247.161.35
-34.87.110.152
-34.87.51.146
-34.87.102.86
-35.247.163.68
 35.240.227.100
-34.142.157.52
-34.87.152.33
 34.124.237.188
-34.143.149.171
+34.143.240.160
+34.87.51.146
 34.87.79.202
-34.87.180.104
+35.247.163.68
+34.87.102.86
 35.198.192.104
+34.126.163.205
+34.124.189.116
+34.143.149.171
+34.87.173.252
+34.142.157.52
+34.87.180.104
+34.87.20.189
+34.87.110.152
+34.87.44.74
+34.87.152.33
+35.197.149.75
+35.247.161.35
 ```

 国内版用户（fastgpt.cn)可以填写下面的 IP 白名单：
-如果仍无响应，可输入命令: `nslookup hzh.sealos.run | awk '/^Address: [0-9]/ {print $2}'` 获取最新 IP

 ```
-47.98.36.227
-118.31.58.217
-121.40.213.28
-120.26.162.94
-223.4.211.186
+47.97.1.240
+121.43.105.217
+121.41.178.7
+121.40.65.187
+47.97.59.172
+101.37.205.32
+120.55.195.90
+120.26.229.115
+120.55.193.112
+47.98.190.173
+112.124.41.79
+121.196.235.183
+121.41.75.88
+121.43.108.48
 ```

 ## 4. 获取AES Key，选择加密方式
--- a/packages/global/common/string/textSplitter.ts
+++ b/packages/global/common/string/textSplitter.ts
@@ -99,7 +99,7 @@ ${mdSplitString}
  5. 标点分割：重叠
 */
 const commonSplit = (props: SplitProps): SplitResponse => {
-  let { text = '', chunkLen, overlapRatio = 0.2, customReg = [] } = props;
+  let { text = '', chunkLen, overlapRatio = 0.15, customReg = [] } = props;

  const splitMarker = 'SPLIT_HERE_SPLIT_HERE';
  const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER';
@@ -113,6 +113,8 @@ const commonSplit = (props: SplitProps): SplitResponse => {
  text = text.replace(/(\r?\n|\r){3,}/g, '\n\n\n');

  // The larger maxLen is, the next sentence is less likely to trigger splitting
+  const markdownIndex = 4;
+  const forbidOverlapIndex = 8;
  const stepReges: { reg: RegExp; maxLen: number }[] = [
    ...customReg.map((text) => ({
      reg: new RegExp(`(${replaceRegChars(text)})`, 'g'),
@@ -122,9 +124,11 @@ const commonSplit = (props: SplitProps): SplitResponse => {
    { reg: /^(##\s[^\n]+\n)/gm, maxLen: chunkLen * 1.4 },
    { reg: /^(###\s[^\n]+\n)/gm, maxLen: chunkLen * 1.6 },
    { reg: /^(####\s[^\n]+\n)/gm, maxLen: chunkLen * 1.8 },
+    { reg: /^(#####\s[^\n]+\n)/gm, maxLen: chunkLen * 1.8 },

    { reg: /([\n]([`~]))/g, maxLen: chunkLen * 4 }, // code block
-    { reg: /([\n](?!\s*[\*\-|>0-9]))/g, maxLen: chunkLen * 2 }, // 增大块，尽可能保证它是一个完整的段落。 (?![\*\-|>`0-9]): markdown special char
+    { reg: /([\n](?=\s*[0-9]+\.))/g, maxLen: chunkLen * 2 }, // 增大块，尽可能保证它是一个完整的段落。 (?![\*\-|>`0-9]): markdown special char
+    { reg: /(\n{2,})/g, maxLen: chunkLen * 1.6 },
    { reg: /([\n])/g, maxLen: chunkLen * 1.2 },
    // ------ There's no overlap on the top
    { reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.2 },
@@ -136,8 +140,9 @@ const commonSplit = (props: SplitProps): SplitResponse => {

  const customRegLen = customReg.length;
  const checkIsCustomStep = (step: number) => step < customRegLen;
-  const checkIsMarkdownSplit = (step: number) => step >= customRegLen && step <= 3 + customRegLen;
-  const checkForbidOverlap = (step: number) => step <= 6 + customRegLen;
+  const checkIsMarkdownSplit = (step: number) => step >= customRegLen && step <= markdownIndex;
+  +customReg.length;
+  const checkForbidOverlap = (step: number) => step <= forbidOverlapIndex + customReg.length;

  // if use markdown title split, Separate record title
  const getSplitTexts = ({ text, step }: { text: string; step: number }) => {
@@ -231,7 +236,7 @@ const commonSplit = (props: SplitProps): SplitResponse => {
      // use slice-chunkLen to split text
      const chunks: string[] = [];
      for (let i = 0; i < text.length; i += chunkLen - overlapLen) {
-        chunks.push(`${parentTitle}${text.slice(i, i + chunkLen)}`);
+        chunks.push(text.slice(i, i + chunkLen));
      }
      return chunks;
    }
@@ -241,7 +246,6 @@ const commonSplit = (props: SplitProps): SplitResponse => {

    const maxLen = splitTexts.length > 1 ? stepReges[step].maxLen : chunkLen;
    const minChunkLen = chunkLen * 0.7;
-    // console.log(splitTexts, stepReges[step].reg);

    const chunks: string[] = [];
    for (let i = 0; i < splitTexts.length; i++) {
@@ -249,12 +253,34 @@ const commonSplit = (props: SplitProps): SplitResponse => {

      const lastTextLen = lastText.length;
      const currentText = item.text;
-      const currentTextLen = currentText.length;
      const newText = lastText + currentText;
-      const newTextLen = lastTextLen + currentTextLen;
+      const newTextLen = newText.length;
+
+      // Markdown 模式下，会强制向下拆分最小块，并再最后一个标题时候，给小块都补充上所有标题（包含父级标题）
+      if (isMarkdownStep) {
+        // split new Text, split chunks must will greater 1 (small lastText)
+        const innerChunks = splitTextRecursively({
+          text: newText,
+          step: step + 1,
+          lastText: '',
+          parentTitle: parentTitle + item.title
+        });
+
+        const lastChunk = innerChunks[innerChunks.length - 1];
+        if (!lastChunk) continue;
+
+        chunks.push(
+          ...innerChunks.map(
+            (chunk) =>
+              step === markdownIndex + customRegLen ? `${parentTitle}${item.title}${chunk}` : chunk // 合并进 Markdown 分块时，需要补标题
+          )
+        );
+
+        continue;
+      }

      // newText is too large(now, The lastText must be smaller than chunkLen)
-      if (newTextLen > maxLen || isMarkdownStep) {
+      if (newTextLen > maxLen) {
        // lastText greater minChunkLen, direct push it to chunks, not add to next chunk. (large lastText)
        if (lastTextLen > minChunkLen) {
          chunks.push(lastText);
@@ -278,15 +304,6 @@ const commonSplit = (props: SplitProps): SplitResponse => {

        if (!lastChunk) continue;

-        if (forbidConcat) {
-          chunks.push(
-            ...innerChunks.map(
-              (chunk) => (step === 3 + customRegLen ? `${parentTitle}${chunk}` : chunk) // 合并进 Markdown 分块时，需要补标题
-            )
-          );
-          continue;
-        }
-
        // last chunk is too small, concat it to lastText(next chunk start)
        if (lastChunk.length < minChunkLen) {
          chunks.push(...innerChunks.slice(0, -1));
@@ -304,11 +321,11 @@ const commonSplit = (props: SplitProps): SplitResponse => {
        continue;
      }

-      // new text is small
+      // New text is small

      // Not overlap
      if (forbidConcat) {
-        chunks.push(`${parentTitle}${item.title}${item.text}`);
+        chunks.push(item.text);
        continue;
      }

--- a/packages/service/core/dataset/collection/schema.ts
+++ b/packages/service/core/dataset/collection/schema.ts
@@ -118,7 +118,7 @@ try {
    {
      unique: true,
      partialFilterExpression: {
-        externalFileId: { $exists: true }
+        externalFileId: { $exists: true, $ne: '' }
      }
    }
  );
--- a/packages/service/core/dataset/training/schema.ts
+++ b/packages/service/core/dataset/training/schema.ts
@@ -51,6 +51,11 @@ const TrainingDataSchema = new Schema({
    type: Date,
    default: () => new Date('2000/1/1')
  },
+  retryCount: {
+    type: Number,
+    default: 5
+  },
+
  model: {
    // ai model
    type: String,
@@ -97,7 +102,7 @@ try {
  // lock training data(teamId); delete training data
  TrainingDataSchema.index({ teamId: 1, datasetId: 1 });
  // get training data and sort
-  TrainingDataSchema.index({ mode: 1, lockTime: 1, weight: -1 });
+  TrainingDataSchema.index({ mode: 1, retryCount: 1, lockTime: 1, weight: -1 });
  TrainingDataSchema.index({ expireAt: 1 }, { expireAfterSeconds: 7 * 24 * 60 * 60 }); // 7 days
 } catch (error) {
  console.log(error);
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -22,7 +22,7 @@ importers:
        version: 13.3.0
      next-i18next:
        specifier: 15.3.0
-        version: 15.3.0(i18next@23.11.5)(next@14.2.5(@babel/core@7.24.9)(babel-plugin-macros@3.1.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))(react-i18next@14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
+        version: 15.3.0(i18next@23.11.5)(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))(react-i18next@14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
      prettier:
        specifier: 3.2.4
        version: 3.2.4
@@ -61,7 +61,7 @@ importers:
        version: 4.0.2
      next:
        specifier: 14.2.5
-        version: 14.2.5(@babel/core@7.24.9)(babel-plugin-macros@3.1.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+        version: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
      openai:
        specifier: 4.61.0
        version: 4.61.0(encoding@0.1.13)
@@ -201,7 +201,7 @@ importers:
        version: 1.4.5-lts.1
      next:
        specifier: 14.2.5
-        version: 14.2.5(@babel/core@7.24.9)(babel-plugin-macros@3.1.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+        version: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
      nextjs-cors:
        specifier: ^2.2.0
        version: 2.2.0(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))
@@ -277,7 +277,7 @@ importers:
        version: 2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)
      '@chakra-ui/next-js':
        specifier: 2.1.5
-        version: 2.1.5(@chakra-ui/react@2.8.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.5(@babel/core@7.24.9)(babel-plugin-macros@3.1.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))(react@18.3.1)
+        version: 2.1.5(@chakra-ui/react@2.8.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))(react@18.3.1)
      '@chakra-ui/react':
        specifier: 2.8.1
        version: 2.8.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -340,7 +340,7 @@ importers:
        version: 4.17.21
      next-i18next:
        specifier: 15.3.0
-        version: 15.3.0(i18next@23.11.5)(next@14.2.5(@babel/core@7.24.9)(babel-plugin-macros@3.1.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))(react-i18next@14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
+        version: 15.3.0(i18next@23.11.5)(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))(react-i18next@14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
      papaparse:
        specifier: ^5.4.1
        version: 5.4.1
@@ -486,6 +486,9 @@ importers:
      json5:
        specifier: ^2.2.3
        version: 2.2.3
+      jsondiffpatch:
+        specifier: ^0.6.0
+        version: 0.6.0
      jsonwebtoken:
        specifier: ^9.0.2
        version: 9.0.2
@@ -700,7 +703,7 @@ importers:
        version: 6.3.4
      ts-jest:
        specifier: ^29.1.0
-        version: 29.2.2(@babel/core@7.24.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.9))(jest@29.7.0(@types/node@20.14.11)(babel-plugin-macros@3.1.0))(typescript@5.5.3)
+        version: 29.2.2(@babel/core@7.24.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.9))(jest@29.7.0(@types/node@20.14.11)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.14.11)(typescript@5.5.3)))(typescript@5.5.3)
      ts-loader:
        specifier: ^9.4.3
        version: 9.5.1(typescript@5.5.3)(webpack@5.92.1)
@@ -3177,8 +3180,8 @@ packages:
  '@tanstack/react-query@4.36.1':
    resolution: {integrity: sha512-y7ySVHFyyQblPl3J3eQBWpXZkliroki3ARnBKsdJchlgt7yJLRDUcf4B8soufgiYt3pEQIkBWBx1N9/ZPIeUWw==}
    peerDependencies:
-      react: ^16.8.0 || ^17.0.0 || ^18.0.0
-      react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0
+      react: 18.3.1
+      react-dom: 18.3.1
      react-native: '*'
    peerDependenciesMeta:
      react-dom:
@@ -3331,6 +3334,9 @@ packages:
  '@types/decompress@4.2.7':
    resolution: {integrity: sha512-9z+8yjKr5Wn73Pt17/ldnmQToaFHZxK0N1GHysuk/JIPT8RIdQeoInM01wWPgypRcvb6VH1drjuFpQ4zmY437g==}

+  '@types/diff-match-patch@1.0.36':
+    resolution: {integrity: sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==}
+
  '@types/eslint-scope@3.7.7':
    resolution: {integrity: sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==}

@@ -4848,6 +4854,9 @@ packages:
  dezalgo@1.0.4:
    resolution: {integrity: sha512-rXSP0bf+5n0Qonsb+SVVfNfIsimO4HEtmnIpPHY8Q1UCzKlQrDMfdobr8nJOOsRgWCyMRqeSBQzmWUMq7zvVig==}

+  diff-match-patch@1.0.5:
+    resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==}
+
  diff-sequences@29.6.3:
    resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==}
    engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0}
@@ -5139,6 +5148,7 @@ packages:
  eslint@8.56.0:
    resolution: {integrity: sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ==}
    engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
+    deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options.
    hasBin: true

  espree@9.6.1:
@@ -6308,6 +6318,11 @@ packages:
  jsonc-parser@3.3.1:
    resolution: {integrity: sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==}

+  jsondiffpatch@0.6.0:
+    resolution: {integrity: sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==}
+    engines: {node: ^18.0.0 || >=20.0.0}
+    hasBin: true
+
  jsonfile@6.1.0:
    resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==}

@@ -10462,6 +10477,14 @@ snapshots:
      next: 14.2.5(@babel/core@7.24.9)(babel-plugin-macros@3.1.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
      react: 18.3.1

+  '@chakra-ui/next-js@2.1.5(@chakra-ui/react@2.8.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))(react@18.3.1)':
+    dependencies:
+      '@chakra-ui/react': 2.8.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
+      '@emotion/cache': 11.11.0
+      '@emotion/react': 11.11.1(@types/react@18.3.1)(react@18.3.1)
+      next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+      react: 18.3.1
+
  '@chakra-ui/number-input@2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)':
    dependencies:
      '@chakra-ui/counter': 2.1.0(react@18.3.1)
@@ -12393,6 +12416,8 @@ snapshots:
    dependencies:
      '@types/node': 22.7.8

+  '@types/diff-match-patch@1.0.36': {}
+
  '@types/eslint-scope@3.7.7':
    dependencies:
      '@types/eslint': 8.56.10
@@ -13203,7 +13228,7 @@ snapshots:

  axios@1.7.7:
    dependencies:
-      follow-redirects: 1.15.9(debug@4.3.7)
+      follow-redirects: 1.15.9
      form-data: 4.0.1
      proxy-from-env: 1.1.0
    transitivePeerDependencies:
@@ -14182,6 +14207,8 @@ snapshots:
      asap: 2.0.6
      wrappy: 1.0.2

+  diff-match-patch@1.0.5: {}
+
  diff-sequences@29.6.3: {}

  diff@4.0.2: {}
@@ -14982,6 +15009,8 @@ snapshots:

  follow-redirects@1.15.6: {}

+  follow-redirects@1.15.9: {}
+
  follow-redirects@1.15.9(debug@4.3.4):
    optionalDependencies:
      debug: 4.3.4
@@ -15044,7 +15073,7 @@ snapshots:
    dependencies:
      react: 18.3.1
      react-dom: 18.3.1(react@18.3.1)
-      tslib: 2.7.0
+      tslib: 2.8.0
    optionalDependencies:
      '@emotion/is-prop-valid': 0.8.8

@@ -16141,6 +16170,12 @@ snapshots:

  jsonc-parser@3.3.1: {}

+  jsondiffpatch@0.6.0:
+    dependencies:
+      '@types/diff-match-patch': 1.0.36
+      chalk: 5.3.0
+      diff-match-patch: 1.0.5
+
  jsonfile@6.1.0:
    dependencies:
      universalify: 2.0.1
@@ -17323,6 +17358,18 @@ snapshots:
      react: 18.3.1
      react-i18next: 14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)

+  next-i18next@15.3.0(i18next@23.11.5)(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8))(react-i18next@14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
+    dependencies:
+      '@babel/runtime': 7.24.8
+      '@types/hoist-non-react-statics': 3.3.5
+      core-js: 3.37.1
+      hoist-non-react-statics: 3.3.2
+      i18next: 23.11.5
+      i18next-fs-backend: 2.3.1
+      next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+      react: 18.3.1
+      react-i18next: 14.1.2(i18next@23.11.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
+
  next@14.2.5(@babel/core@7.24.9)(babel-plugin-macros@3.1.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8):
    dependencies:
      '@next/env': 14.2.5
@@ -17349,10 +17396,36 @@ snapshots:
      - '@babel/core'
      - babel-plugin-macros

+  next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8):
+    dependencies:
+      '@next/env': 14.2.5
+      '@swc/helpers': 0.5.5
+      busboy: 1.6.0
+      caniuse-lite: 1.0.30001669
+      graceful-fs: 4.2.11
+      postcss: 8.4.31
+      react: 18.3.1
+      react-dom: 18.3.1(react@18.3.1)
+      styled-jsx: 5.1.1(react@18.3.1)
+    optionalDependencies:
+      '@next/swc-darwin-arm64': 14.2.5
+      '@next/swc-darwin-x64': 14.2.5
+      '@next/swc-linux-arm64-gnu': 14.2.5
+      '@next/swc-linux-arm64-musl': 14.2.5
+      '@next/swc-linux-x64-gnu': 14.2.5
+      '@next/swc-linux-x64-musl': 14.2.5
+      '@next/swc-win32-arm64-msvc': 14.2.5
+      '@next/swc-win32-ia32-msvc': 14.2.5
+      '@next/swc-win32-x64-msvc': 14.2.5
+      sass: 1.77.8
+    transitivePeerDependencies:
+      - '@babel/core'
+      - babel-plugin-macros
+
  nextjs-cors@2.2.0(next@14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)):
    dependencies:
      cors: 2.8.5
-      next: 14.2.5(@babel/core@7.24.9)(babel-plugin-macros@3.1.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)
+      next: 14.2.5(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.77.8)

  nextjs-node-loader@1.1.5(webpack@5.92.1):
    dependencies:
@@ -18410,7 +18483,7 @@ snapshots:
    dependencies:
      chokidar: 3.6.0
      immutable: 4.3.6
-      source-map-js: 1.2.0
+      source-map-js: 1.2.1

  sax@1.4.1: {}

@@ -18755,6 +18828,11 @@ snapshots:
      '@babel/core': 7.24.9
      babel-plugin-macros: 3.1.0

+  styled-jsx@5.1.1(react@18.3.1):
+    dependencies:
+      client-only: 0.0.1
+      react: 18.3.1
+
  stylis@4.2.0: {}

  stylis@4.3.2: {}
@@ -18956,6 +19034,25 @@ snapshots:

  ts-dedent@2.2.0: {}

+  ts-jest@29.2.2(@babel/core@7.24.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.9))(jest@29.7.0(@types/node@20.14.11)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.14.11)(typescript@5.5.3)))(typescript@5.5.3):
+    dependencies:
+      bs-logger: 0.2.6
+      ejs: 3.1.10
+      fast-json-stable-stringify: 2.1.0
+      jest: 29.7.0(@types/node@20.14.11)(babel-plugin-macros@3.1.0)(ts-node@10.9.2(@types/node@20.14.11)(typescript@5.5.3))
+      jest-util: 29.7.0
+      json5: 2.2.3
+      lodash.memoize: 4.1.2
+      make-error: 1.3.6
+      semver: 7.6.3
+      typescript: 5.5.3
+      yargs-parser: 21.1.1
+    optionalDependencies:
+      '@babel/core': 7.24.9
+      '@jest/transform': 29.7.0
+      '@jest/types': 29.6.3
+      babel-jest: 29.7.0(@babel/core@7.24.9)
+
  ts-jest@29.2.2(@babel/core@7.24.9)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.24.9))(jest@29.7.0(@types/node@20.14.11)(babel-plugin-macros@3.1.0))(typescript@5.5.3):
    dependencies:
      bs-logger: 0.2.6
--- a/projects/app/.env.template
+++ b/projects/app/.env.template
@@ -33,7 +33,7 @@ MILVUS_TOKEN=133964348b00b4b4e4b51bef680a61350950385c8c64a3ec16b1ab92d3c67dcc4e0
 SANDBOX_URL=http://localhost:3001
 # 商业版地址
 PRO_URL=
-# 页面的地址，用于自动补全相对路径资源的 domain
+# 页面的地址，用于自动补全相对路径资源的 domain，注意后面不要跟 /
 FE_DOMAIN=http://localhost:3000
 # 二级路由，需要打包时候就确定
 # NEXT_PUBLIC_BASE_URL=/fastai
--- a/projects/app/package.json
+++ b/projects/app/package.json
@@ -42,6 +42,7 @@
    "jest": "^29.5.0",
    "js-yaml": "^4.1.0",
    "json5": "^2.2.3",
+    "jsondiffpatch": "^0.6.0",
    "jsonwebtoken": "^9.0.2",
    "lodash": "^4.17.21",
    "mermaid": "^10.2.3",
--- a/projects/app/src/pages/api/core/dataset/update.ts
+++ b/projects/app/src/pages/api/core/dataset/update.ts
@@ -9,7 +9,7 @@ import {
 } from '@fastgpt/global/support/permission/constant';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
-import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { DatasetTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { ClientSession } from 'mongoose';
 import { parseParentIdInMongo } from '@fastgpt/global/common/parentFolder/utils';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
@@ -21,6 +21,7 @@ import {
 import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
 import { TeamWritePermissionVal } from '@fastgpt/global/support/permission/user/constant';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';

 export type DatasetUpdateQuery = {};
 export type DatasetUpdateResponse = any;
@@ -84,6 +85,12 @@ async function handler(

  const isFolder = dataset.type === DatasetTypeEnum.folder;

+  updateTraining({
+    teamId: dataset.teamId,
+    datasetId: id,
+    agentModel: agentModel?.model
+  });
+
  const onUpdate = async (session?: ClientSession) => {
    await MongoDataset.findByIdAndUpdate(
      id,
@@ -137,3 +144,29 @@ async function handler(
  }
 }
 export default NextAPI(handler);
+
+async function updateTraining({
+  teamId,
+  datasetId,
+  agentModel
+}: {
+  teamId: string;
+  datasetId: string;
+  agentModel?: string;
+}) {
+  if (!agentModel) return;
+
+  await MongoDatasetTraining.updateMany(
+    {
+      teamId,
+      datasetId,
+      mode: { $in: [TrainingModeEnum.qa, TrainingModeEnum.auto] }
+    },
+    {
+      $set: {
+        model: agentModel,
+        retryCount: 5
+      }
+    }
+  );
+}
--- a/projects/app/src/pages/app/detail/components/Plugin/Header.tsx
+++ b/projects/app/src/pages/app/detail/components/Plugin/Header.tsx
@@ -34,6 +34,7 @@ import {
  WorkflowInitContext
 } from '../WorkflowComponents/context/workflowInitContext';
 import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
+import { getAppConfigByDiff } from '@/web/core/app/diff';

 const Header = () => {
  const { t } = useTranslation();
@@ -51,16 +52,19 @@ const Header = () => {

  const nodes = useContextSelector(WorkflowInitContext, (v) => v.nodes);
  const edges = useContextSelector(WorkflowNodeEdgeContext, (v) => v.edges);
-  const {
-    flowData2StoreData,
-    flowData2StoreDataAndCheck,
-    setWorkflowTestData,
-    past,
-    future,
-    setPast,
-    onSwitchTmpVersion,
-    onSwitchCloudVersion
-  } = useContextSelector(WorkflowContext, (v) => v);
+
+  const flowData2StoreData = useContextSelector(WorkflowContext, (v) => v.flowData2StoreData);
+  const flowData2StoreDataAndCheck = useContextSelector(
+    WorkflowContext,
+    (v) => v.flowData2StoreDataAndCheck
+  );
+  const setWorkflowTestData = useContextSelector(WorkflowContext, (v) => v.setWorkflowTestData);
+  const past = useContextSelector(WorkflowContext, (v) => v.past);
+  const future = useContextSelector(WorkflowContext, (v) => v.future);
+  const setPast = useContextSelector(WorkflowContext, (v) => v.setPast);
+  const onSwitchTmpVersion = useContextSelector(WorkflowContext, (v) => v.onSwitchTmpVersion);
+  const onSwitchCloudVersion = useContextSelector(WorkflowContext, (v) => v.onSwitchCloudVersion);
+
  const showHistoryModal = useContextSelector(WorkflowEventContext, (v) => v.showHistoryModal);
  const setShowHistoryModal = useContextSelector(
    WorkflowEventContext,
@@ -76,15 +80,20 @@ const Header = () => {
        [...future].reverse().find((snapshot) => snapshot.isSaved) ||
        past.find((snapshot) => snapshot.isSaved);

+      const initialState = past[past.length - 1]?.state;
+      const savedSnapshotState = getAppConfigByDiff(initialState, savedSnapshot?.diff);
+
      const val = compareSnapshot(
+        // nodes of the saved snapshot
        {
-          nodes: savedSnapshot?.nodes,
-          edges: savedSnapshot?.edges,
-          chatConfig: savedSnapshot?.chatConfig
+          nodes: savedSnapshotState?.nodes,
+          edges: savedSnapshotState?.edges,
+          chatConfig: savedSnapshotState?.chatConfig
        },
+        // nodes of the current canvas
        {
-          nodes: nodes,
-          edges: edges,
+          nodes,
+          edges,
          chatConfig: appDetail.chatConfig
        }
      );
@@ -132,8 +141,6 @@ const Header = () => {

  const onBack = useCallback(async () => {
    try {
-      localStorage.removeItem(`${appDetail._id}-past`);
-      localStorage.removeItem(`${appDetail._id}-future`);
      router.push({
        pathname: '/app/list',
        query: {
@@ -142,7 +149,7 @@ const Header = () => {
        }
      });
    } catch (error) {}
-  }, [appDetail._id, appDetail.parentId, lastAppListRouteType, router]);
+  }, [appDetail.parentId, lastAppListRouteType, router]);

  const Render = useMemo(() => {
    return (
--- a/projects/app/src/pages/app/detail/components/SimpleApp/Edit.tsx
+++ b/projects/app/src/pages/app/detail/components/SimpleApp/Edit.tsx
@@ -17,16 +17,44 @@ import styles from './styles.module.scss';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import { useTranslation } from 'next-i18next';
 import { onSaveSnapshotFnType, SimpleAppSnapshotType } from './useSnapshots';
+import { getAppConfigByDiff, getAppDiffConfig } from '@/web/core/app/diff';
+import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
+
+const convertOldFormatHistory = (past: SimpleAppSnapshotType[]) => {
+  const baseState = past[past.length - 1].appForm;
+
+  return past.map((item, index) => {
+    if (index === past.length - 1) {
+      return {
+        title: item.title,
+        isSaved: item.isSaved,
+        state: baseState
+      };
+    }
+
+    const currentState = item.appForm;
+
+    const diff = getAppDiffConfig(baseState, currentState);
+
+    return {
+      title: item.title || formatTime2YMDHMS(new Date()),
+      isSaved: item.isSaved,
+      diff
+    };
+  });
+};

 const Edit = ({
  appForm,
  setAppForm,
  past,
+  setPast,
  saveSnapshot
 }: {
  appForm: AppSimpleEditFormType;
  setAppForm: React.Dispatch<React.SetStateAction<AppSimpleEditFormType>>;
  past: SimpleAppSnapshotType[];
+  setPast: (value: React.SetStateAction<SimpleAppSnapshotType[]>) => void;
  saveSnapshot: onSaveSnapshotFnType;
 }) => {
  const { isPc } = useSystem();
@@ -39,9 +67,26 @@ const Edit = ({
    // show selected dataset
    loadAllDatasets();

+    if (appDetail.version !== 'v2') {
+      return setAppForm(
+        appWorkflow2Form({
+          nodes: v1Workflow2V2((appDetail.modules || []) as any)?.nodes,
+          chatConfig: appDetail.chatConfig
+        })
+      );
+    }
+
    // Get the latest snapshot
-    if (past?.[0]?.appForm) {
-      return setAppForm(past[0].appForm);
+    if (past?.[0]?.diff) {
+      const pastState = getAppConfigByDiff(past[past.length - 1].state, past[0].diff);
+
+      return setAppForm(pastState);
+    } else if (past && past.length > 0 && past?.every((item) => item.appForm)) {
+      // 格式化成 diff
+      const newPast = convertOldFormatHistory(past);
+
+      setPast(newPast);
+      return setAppForm(getAppConfigByDiff(newPast[newPast.length - 1].state, newPast[0].diff));
    }

    const appForm = appWorkflow2Form({
@@ -59,15 +104,6 @@ const Edit = ({
    }

    setAppForm(appForm);
-
-    if (appDetail.version !== 'v2') {
-      setAppForm(
-        appWorkflow2Form({
-          nodes: v1Workflow2V2((appDetail.modules || []) as any)?.nodes,
-          chatConfig: appDetail.chatConfig
-        })
-      );
-    }
  });

  return (
--- a/projects/app/src/pages/app/detail/components/SimpleApp/Header.tsx
+++ b/projects/app/src/pages/app/detail/components/SimpleApp/Header.tsx
@@ -1,4 +1,4 @@
-import React, { useCallback, useState } from 'react';
+import React, { useCallback, useMemo, useState } from 'react';
 import { useContextSelector } from 'use-context-selector';
 import { AppContext } from '../context';
 import FolderPath from '@/components/common/folder/Path';
@@ -29,6 +29,7 @@ import {
 } from './useSnapshots';
 import PublishHistories from '../PublishHistoriesSlider';
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
+import { getAppConfigByDiff } from '@/web/core/app/diff';

 const Header = ({
  forbiddenSaveSnapshot,
@@ -48,7 +49,11 @@ const Header = ({
  const { t } = useTranslation();
  const { isPc } = useSystem();
  const router = useRouter();
-  const { appId, onSaveApp, currentTab } = useContextSelector(AppContext, (v) => v);
+  const appId = useContextSelector(AppContext, (v) => v.appId);
+  const onSaveApp = useContextSelector(AppContext, (v) => v.onSaveApp);
+  const currentTab = useContextSelector(AppContext, (v) => v.currentTab);
+  const appLatestVersion = useContextSelector(AppContext, (v) => v.appLatestVersion);
+
  const { lastAppListRouteType } = useSystemStore();
  const { allDatasets } = useDatasetStore();

@@ -102,9 +107,19 @@ const Header = ({
  const [isShowHistories, { setTrue: setIsShowHistories, setFalse: closeHistories }] =
    useBoolean(false);

+  const initialAppForm = useMemo(
+    () =>
+      appWorkflow2Form({
+        nodes: appLatestVersion?.nodes || [],
+        chatConfig: appLatestVersion?.chatConfig || {}
+      }),
+    [appLatestVersion]
+  );
+
  const onSwitchTmpVersion = useCallback(
    (data: SimpleAppSnapshotType, customTitle: string) => {
-      setAppForm(data.appForm);
+      const pastState = getAppConfigByDiff(initialAppForm, data.diff);
+      setAppForm(pastState);

      // Remove multiple "copy-"
      const copyText = t('app:version_copy');
@@ -112,11 +127,11 @@ const Header = ({
      const title = customTitle.replace(regex, `$1`);

      return saveSnapshot({
-        appForm: data.appForm,
+        appForm: pastState,
        title
      });
    },
-    [saveSnapshot, setAppForm, t]
+    [initialAppForm, saveSnapshot, setAppForm, t]
  );
  const onSwitchCloudVersion = useCallback(
    (appVersion: AppVersionSchemaType) => {
@@ -143,7 +158,8 @@ const Header = ({
  useDebounceEffect(
    () => {
      const savedSnapshot = past.find((snapshot) => snapshot.isSaved);
-      const val = compareSimpleAppSnapshot(savedSnapshot?.appForm, appForm);
+      const pastState = getAppConfigByDiff(initialAppForm, savedSnapshot?.diff);
+      const val = compareSimpleAppSnapshot(pastState, appForm);
      setIsPublished(val);
    },
    [past, allDatasets],
--- a/projects/app/src/pages/app/detail/components/SimpleApp/index.tsx
+++ b/projects/app/src/pages/app/detail/components/SimpleApp/index.tsx
@@ -49,7 +49,13 @@ const SimpleEdit = () => {
        saveSnapshot={saveSnapshot}
      />
      {currentTab === TabEnum.appEdit ? (
-        <Edit appForm={appForm} setAppForm={setAppForm} past={past} saveSnapshot={saveSnapshot} />
+        <Edit
+          appForm={appForm}
+          setAppForm={setAppForm}
+          past={past}
+          setPast={setPast}
+          saveSnapshot={saveSnapshot}
+        />
      ) : (
        <Box flex={'1 0 0'} h={0} mt={[4, 0]}>
          {currentTab === TabEnum.publish && <PublishChannel />}
--- a/projects/app/src/pages/app/detail/components/SimpleApp/useSnapshots.tsx
+++ b/projects/app/src/pages/app/detail/components/SimpleApp/useSnapshots.tsx
@@ -3,14 +3,19 @@ import { SetStateAction, useEffect, useRef } from 'react';
 import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
 import { AppSimpleEditFormType } from '@fastgpt/global/core/app/type';
 import { isEqual } from 'lodash';
+import { getAppDiffConfig } from '@/web/core/app/diff';

 export type SimpleAppSnapshotType = {
-  appForm: AppSimpleEditFormType;
+  diff?: Record<string, any>;
  title: string;
  isSaved?: boolean;
+  state?: AppSimpleEditFormType;
+
+  // old format
+  appForm?: AppSimpleEditFormType;
 };
 export type onSaveSnapshotFnType = (props: {
-  appForm: AppSimpleEditFormType;
+  appForm: AppSimpleEditFormType; // Current edited app form data
  title?: string;
  isSaved?: boolean;
 }) => Promise<boolean>;
@@ -56,7 +61,7 @@ export const compareSimpleAppSnapshot = (

 export const useSimpleAppSnapshots = (appId: string) => {
  const forbiddenSaveSnapshot = useRef(false);
-  const [past, setPast] = useLocalStorageState<SimpleAppSnapshotType[]>(`${appId}-past-simple`, {
+  const [past, setPast] = useLocalStorageState<SimpleAppSnapshotType[]>(`${appId}-past`, {
    defaultValue: []
  }) as [SimpleAppSnapshotType[], (value: SetStateAction<SimpleAppSnapshotType[]>) => void];

@@ -66,26 +71,47 @@ export const useSimpleAppSnapshots = (appId: string) => {
      return false;
    }

-    const pastState = past[0];
+    if (past.length === 0) {
+      setPast([
+        {
+          title: title || formatTime2YMDHMS(new Date()),
+          isSaved,
+          state: appForm
+        }
+      ]);
+      return true;
+    }

-    const isPastEqual = compareSimpleAppSnapshot(pastState?.appForm, appForm);
-    if (isPastEqual) return false;
+    const lastPast = past[past.length - 1];
+    if (!lastPast?.state) return false;

-    setPast((past) => [
-      {
-        appForm,
+    // Get the diff between the current app form data and the initial state
+    const diff = getAppDiffConfig(lastPast.state, appForm);
+
+    // If the diff is the same as the previous snapshot, do not save
+    if (past[0].diff && isEqual(past[0].diff, diff)) return false;
+
+    setPast((past) => {
+      const newPast = {
+        diff,
        title: title || formatTime2YMDHMS(new Date()),
        isSaved
-      },
-      ...past.slice(0, 199)
-    ]);
+      };
+
+      if (past.length >= 100) {
+        return [newPast, ...past.slice(0, 98), lastPast];
+      }
+      return [newPast, ...past];
+    });
    return true;
  });

  // remove other app's snapshot
  useEffect(() => {
    const keys = Object.keys(localStorage);
-    const snapshotKeys = keys.filter((key) => key.endsWith('-past-simple'));
+    const snapshotKeys = keys.filter(
+      (key) => key.endsWith('-past') || key.endsWith('-past-simple')
+    );
    snapshotKeys.forEach((key) => {
      const keyAppId = key.split('-')[0];
      if (keyAppId !== appId) {
@@ -94,6 +120,20 @@ export const useSimpleAppSnapshots = (appId: string) => {
    });
  }, [appId]);

+  // 旧的编辑记录，直接重置到新的变量中
+  const [oldPast, setOldPast] = useLocalStorageState<SimpleAppSnapshotType[]>(
+    `${appId}-past-simple`,
+    {}
+  );
+  useEffect(() => {
+    if (oldPast && oldPast.length > 0) {
+      setPast(past);
+      setOldPast([]);
+      // refresh page
+      window.location.reload();
+    }
+  }, [oldPast]);
+
  return { forbiddenSaveSnapshot, past, setPast, saveSnapshot };
 };

--- a/projects/app/src/pages/app/detail/components/Workflow/Header.tsx
+++ b/projects/app/src/pages/app/detail/components/Workflow/Header.tsx
@@ -34,6 +34,7 @@ import {
  WorkflowInitContext
 } from '../WorkflowComponents/context/workflowInitContext';
 import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
+import { getAppConfigByDiff } from '@/web/core/app/diff';

 const Header = () => {
  const { t } = useTranslation();
@@ -55,16 +56,19 @@ const Header = () => {

  const nodes = useContextSelector(WorkflowInitContext, (v) => v.nodes);
  const edges = useContextSelector(WorkflowNodeEdgeContext, (v) => v.edges);
-  const {
-    flowData2StoreData,
-    flowData2StoreDataAndCheck,
-    setWorkflowTestData,
-    past,
-    future,
-    setPast,
-    onSwitchTmpVersion,
-    onSwitchCloudVersion
-  } = useContextSelector(WorkflowContext, (v) => v);
+
+  const flowData2StoreData = useContextSelector(WorkflowContext, (v) => v.flowData2StoreData);
+  const flowData2StoreDataAndCheck = useContextSelector(
+    WorkflowContext,
+    (v) => v.flowData2StoreDataAndCheck
+  );
+  const setWorkflowTestData = useContextSelector(WorkflowContext, (v) => v.setWorkflowTestData);
+  const past = useContextSelector(WorkflowContext, (v) => v.past);
+  const future = useContextSelector(WorkflowContext, (v) => v.future);
+  const setPast = useContextSelector(WorkflowContext, (v) => v.setPast);
+  const onSwitchTmpVersion = useContextSelector(WorkflowContext, (v) => v.onSwitchTmpVersion);
+  const onSwitchCloudVersion = useContextSelector(WorkflowContext, (v) => v.onSwitchCloudVersion);
+
  const showHistoryModal = useContextSelector(WorkflowEventContext, (v) => v.showHistoryModal);
  const setShowHistoryModal = useContextSelector(
    WorkflowEventContext,
@@ -81,11 +85,14 @@ const Header = () => {
        [...future].reverse().find((snapshot) => snapshot.isSaved) ||
        past.find((snapshot) => snapshot.isSaved);

+      const initialState = past[past.length - 1]?.state;
+      const savedSnapshotState = getAppConfigByDiff(initialState, savedSnapshot?.diff);
+
      const val = compareSnapshot(
        {
-          nodes: savedSnapshot?.nodes,
-          edges: savedSnapshot?.edges,
-          chatConfig: savedSnapshot?.chatConfig
+          nodes: savedSnapshotState?.nodes,
+          edges: savedSnapshotState?.edges,
+          chatConfig: savedSnapshotState?.chatConfig
        },
        {
          nodes: nodes,
@@ -137,8 +144,6 @@ const Header = () => {

  const onBack = useCallback(async () => {
    try {
-      localStorage.removeItem(`${appDetail._id}-past`);
-      localStorage.removeItem(`${appDetail._id}-future`);
      router.push({
        pathname: '/app/list',
        query: {
@@ -147,7 +152,7 @@ const Header = () => {
        }
      });
    } catch (error) {}
-  }, [appDetail._id, appDetail.parentId, lastAppListRouteType, router]);
+  }, [appDetail.parentId, lastAppListRouteType, router]);

  const Render = useMemo(() => {
    return (
--- a/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/components/ContextMenu.tsx
+++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/Flow/components/ContextMenu.tsx
@@ -5,7 +5,6 @@ import { useTranslation } from 'react-i18next';
 import { nodeTemplate2FlowNode } from '@/web/core/workflow/utils';
 import { CommentNode } from '@fastgpt/global/core/workflow/template/system/comment';
 import { useContextSelector } from 'use-context-selector';
-import { WorkflowContext } from '../../context';
 import { useReactFlow } from 'reactflow';
 import { WorkflowNodeEdgeContext } from '../../context/workflowInitContext';
 import { WorkflowEventContext } from '../../context/workflowEventContext';
--- a/projects/app/src/pages/app/detail/components/WorkflowComponents/context/index.tsx
+++ b/projects/app/src/pages/app/detail/components/WorkflowComponents/context/index.tsx
@@ -1,7 +1,7 @@
 import { postWorkflowDebug } from '@/web/core/workflow/api';
 import {
  checkWorkflowNodeAndConnection,
-  compareSnapshot,
+  simplifyWorkflowNodes,
  storeEdgesRenderEdge,
  storeNode2FlowNode
 } from '@/web/core/workflow/utils';
@@ -37,10 +37,11 @@ import { useDisclosure } from '@chakra-ui/react';
 import { uiWorkflow2StoreWorkflow } from '../utils';
 import { useTranslation } from 'next-i18next';
 import { formatTime2YMDHMS, formatTime2YMDHMW } from '@fastgpt/global/common/string/time';
-import { cloneDeep } from 'lodash';
+import { cloneDeep, isEqual } from 'lodash';
 import { AppVersionSchemaType } from '@fastgpt/global/core/app/version';
 import WorkflowInitContextProvider, { WorkflowNodeEdgeContext } from './workflowInitContext';
 import WorkflowEventContextProvider from './workflowEventContext';
+import { getAppConfigByDiff, getAppDiffConfig } from '@/web/core/app/diff';

 /* 
  Context
@@ -67,14 +68,22 @@ export const ReactFlowCustomProvider = ({
  );
 };

-type OnChange<ChangesType> = (changes: ChangesType[]) => void;
-
 export type WorkflowSnapshotsType = {
+  diff?: any;
+  title: string;
+  isSaved?: boolean;
+  state?: WorkflowStateType;
+
+  // old format
+  nodes?: Node[];
+  edges?: Edge[];
+  chatConfig?: AppChatConfigType;
+};
+
+export type WorkflowStateType = {
  nodes: Node[];
  edges: Edge[];
-  title: string;
  chatConfig: AppChatConfigType;
-  isSaved?: boolean;
 };

 type WorkflowContextType = {
@@ -751,7 +760,7 @@ const WorkflowContextProvider = ({
    defaultValue: []
  }) as [WorkflowSnapshotsType[], (value: SetStateAction<WorkflowSnapshotsType[]>) => void];

-  const resetSnapshot = useMemoizedFn((state: Omit<WorkflowSnapshotsType, 'title' | 'isSaved'>) => {
+  const resetSnapshot = useMemoizedFn((state: WorkflowStateType) => {
    setNodes(state.nodes);
    setEdges(state.edges);
    setAppDetail((detail) => ({
@@ -759,70 +768,60 @@ const WorkflowContextProvider = ({
      chatConfig: state.chatConfig
    }));
  });
-  const pushPastSnapshot = useMemoizedFn(
-    ({
-      pastNodes,
-      pastEdges,
-      customTitle,
-      chatConfig,
-      isSaved
-    }: {
-      pastNodes: Node[];
-      pastEdges: Edge[];
-      customTitle?: string;
-      chatConfig: AppChatConfigType;
-      isSaved?: boolean;
-    }) => {
-      if (!pastNodes || !pastEdges || !chatConfig) return false;

+  const pushPastSnapshot = useMemoizedFn(
+    ({ pastNodes, pastEdges, chatConfig, customTitle, isSaved }) => {
+      if (!pastNodes || !pastEdges || !chatConfig) return false;
      if (forbiddenSaveSnapshot.current) {
        forbiddenSaveSnapshot.current = false;
        return false;
      }

-      const pastState = past[0];
-      const isPastEqual = compareSnapshot(
-        {
-          nodes: pastNodes,
-          edges: pastEdges,
-          chatConfig: chatConfig
-        },
-        {
-          nodes: pastState?.nodes,
-          edges: pastState?.edges,
-          chatConfig: pastState?.chatConfig
-        }
-      );
+      // Get initial state
+      const lastSnapshot = past[past.length - 1];
+      if (!lastSnapshot?.state) return false;

-      if (isPastEqual) return false;
+      // Create current state object
+      const newState = {
+        nodes: simplifyWorkflowNodes(pastNodes),
+        edges: pastEdges,
+        chatConfig
+      };
+
+      // Calculate diff from initial state
+      const diff = getAppDiffConfig(lastSnapshot.state, newState);
+      if (past[0].diff && isEqual(past[0].diff, diff)) return false;

      setFuture([]);
-      setPast((past) => [
-        {
-          nodes: pastNodes,
-          edges: pastEdges,
+      setPast((past) => {
+        const newPast = {
+          diff,
          title: customTitle || formatTime2YMDHMS(new Date()),
-          chatConfig,
          isSaved
-        },
-        ...past.slice(0, 199)
-      ]);
+        };
+        if (past.length >= 100) {
+          return [newPast, ...past.slice(0, 98), lastSnapshot];
+        }
+        return [newPast, ...past];
+      });

      return true;
    }
  );
+
  const onSwitchTmpVersion = useMemoizedFn((params: WorkflowSnapshotsType, customTitle: string) => {
    // Remove multiple "copy-"
    const copyText = t('app:version_copy');
    const regex = new RegExp(`(${copyText}-)\\1+`, 'g');
    const title = customTitle.replace(regex, `$1`);
+    const pastState = getAppConfigByDiff(past[past.length - 1].state, params.diff);

-    resetSnapshot(params);
+    resetSnapshot(pastState);

    return pushPastSnapshot({
-      pastNodes: params.nodes,
-      pastEdges: params.edges,
-      chatConfig: params.chatConfig,
+      pastNodes: pastState.nodes,
+      pastEdges: pastState.edges,
+      chatConfig: pastState.chatConfig,
      customTitle: title
    });
  });
@@ -848,15 +847,19 @@ const WorkflowContextProvider = ({
    if (past[1]) {
      setFuture((future) => [past[0], ...future]);
      setPast((past) => past.slice(1));
-      resetSnapshot(past[1]);
+      const pastState = getAppConfigByDiff(past[past.length - 1].state, past[1].diff);
+      resetSnapshot(pastState);
    }
  });
  const redo = useMemoizedFn(() => {
-    const futureState = future[0];
+    if (!future[0]) return;
+
+    const futureState = getAppConfigByDiff(past[past.length - 1].state, future[0].diff);

    if (futureState) {
      setPast((past) => [future[0], ...past]);
      setFuture((future) => future.slice(1));
+
      resetSnapshot(futureState);
    }
  });
@@ -874,44 +877,79 @@ const WorkflowContextProvider = ({
  }, [appId]);

  const initData = useCallback(
-    async (e: Parameters<WorkflowContextType['initData']>[0], isInit?: boolean) => {
-      // Refresh web page, load init
+    async (
+      e: {
+        nodes: StoreNodeItemType[];
+        edges: StoreEdgeItemType[];
+        chatConfig?: AppChatConfigType;
+      },
+      isInit?: boolean
+    ) => {
+      const nodes = e.nodes?.map((item) => storeNode2FlowNode({ item, t })) || [];
+      const edges = e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })) || [];
+
+      const initialState = {
+        nodes: simplifyWorkflowNodes(nodes),
+        edges,
+        chatConfig: e.chatConfig || appDetail.chatConfig
+      };
+
      if (isInit && past.length > 0) {
-        return resetSnapshot(past[0]);
+        // new format
+        if (past[0].diff && past[past.length - 1].state) {
+          const targetState = getAppConfigByDiff(
+            past[past.length - 1].state,
+            past[0].diff
+          ) as WorkflowStateType;
+
+          setNodes(targetState.nodes);
+          setEdges(targetState.edges);
+          setAppDetail((state) => ({
+            ...state,
+            chatConfig: targetState.chatConfig
+          }));
+          return;
+        }
+
+        // 适配旧的编辑记录（4.8.15去除）
+        if (past.every((item) => item.nodes)) {
+          const newPast = convertOldFormatHistory(past);
+
+          setPast(newPast);
+
+          const latestState = getAppConfigByDiff(
+            newPast[newPast.length - 1].state,
+            newPast[0].diff
+          ) as WorkflowStateType;
+
+          setNodes(latestState.nodes);
+          setEdges(latestState.edges);
+          setAppDetail((state) => ({
+            ...state,
+            chatConfig: latestState.chatConfig
+          }));
+          return;
+        }
      }
-      // If it is the initial data, save the initial snapshot
+
+      setNodes(nodes);
+      setEdges(edges);
+      if (e.chatConfig) {
+        setAppDetail((state) => ({ ...state, chatConfig: e.chatConfig as AppChatConfigType }));
+      }
+
      if (isInit && past.length === 0) {
-        pushPastSnapshot({
-          pastNodes: e.nodes?.map((item) => storeNode2FlowNode({ item, t })) || [],
-          pastEdges: e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })) || [],
-          customTitle: t(`app:app.version_initial`),
-          chatConfig: appDetail.chatConfig,
-          isSaved: true
-        });
+        setPast([
+          {
+            title: t(`app:app.version_initial`),
+            isSaved: true,
+            state: initialState
+          }
+        ]);
        forbiddenSaveSnapshot.current = true;
      }
-
-      setNodes(e.nodes?.map((item) => storeNode2FlowNode({ item, t })) || []);
-      setEdges(e.edges?.map((item) => storeEdgesRenderEdge({ edge: item })) || []);
-
-      const chatConfig = e.chatConfig;
-      if (chatConfig) {
-        setAppDetail((state) => ({
-          ...state,
-          chatConfig
-        }));
-      }
    },
-    [
-      appDetail.chatConfig,
-      past,
-      resetSnapshot,
-      pushPastSnapshot,
-      setAppDetail,
-      setEdges,
-      setNodes,
-      t
-    ]
+    [appDetail.chatConfig, past, setAppDetail, setEdges, setNodes, setPast, t]
  );

  const value = useMemo(
@@ -997,3 +1035,36 @@ const WorkflowContextProvider = ({
  );
 };
 export default React.memo(WorkflowContextProvider);
+
+// Convert old history format to new format
+const convertOldFormatHistory = (past: WorkflowSnapshotsType[]) => {
+  const baseState = {
+    nodes: past[past.length - 1].state?.nodes || [],
+    edges: past[past.length - 1].state?.edges || [],
+    chatConfig: past[past.length - 1].state?.chatConfig || {}
+  };
+
+  return past.map((item, index) => {
+    if (index === past.length - 1) {
+      return {
+        title: item.title,
+        isSaved: item.isSaved,
+        state: baseState
+      };
+    }
+
+    const currentState = {
+      nodes: item.nodes || [],
+      edges: item.edges || [],
+      chatConfig: item.chatConfig || {}
+    };
+
+    const diff = getAppDiffConfig(baseState, currentState);
+
+    return {
+      title: item.title || formatTime2YMDHMS(new Date()),
+      isSaved: item.isSaved,
+      diff
+    };
+  });
+};
--- a/projects/app/src/pages/chat/share.tsx
+++ b/projects/app/src/pages/chat/share.tsx
@@ -385,7 +385,7 @@ const Render = (props: Props) => {
  const [isLoaded, setIsLoaded] = useState(false);

  const contextParams = useMemo(() => {
-    return { shareId, outLinkUid: authToken || localUId || customUid };
+    return { shareId, outLinkUid: authToken || customUid || localUId };
  }, [authToken, customUid, localUId, shareId]);

  useMount(() => {
--- a/projects/app/src/service/events/generateQA.ts
+++ b/projects/app/src/service/events/generateQA.ts
@@ -39,11 +39,13 @@ export async function generateQA(): Promise<any> {
    try {
      const data = await MongoDatasetTraining.findOneAndUpdate(
        {
-          lockTime: { $lte: addMinutes(new Date(), -6) },
-          mode: TrainingModeEnum.qa
+          mode: TrainingModeEnum.qa,
+          retryCount: { $gte: 0 },
+          lockTime: { $lte: addMinutes(new Date(), -6) }
        },
        {
-          lockTime: new Date()
+          lockTime: new Date(),
+          $inc: { retryCount: -1 }
        }
      )
        .select({
--- a/projects/app/src/service/events/generateVector.ts
+++ b/projects/app/src/service/events/generateVector.ts
@@ -39,10 +39,12 @@ export async function generateVector(): Promise<any> {
      const data = await MongoDatasetTraining.findOneAndUpdate(
        {
          mode: TrainingModeEnum.chunk,
-          lockTime: { $lte: addMinutes(new Date(), -1) }
+          retryCount: { $gte: 0 },
+          lockTime: { $lte: addMinutes(new Date(), -6) }
        },
        {
-          lockTime: new Date()
+          lockTime: new Date(),
+          $inc: { retryCount: -1 }
        }
      ).select({
        _id: 1,
--- a/projects/app/src/web/core/app/diff.ts
+++ b/projects/app/src/web/core/app/diff.ts
@@ -0,0 +1,23 @@
+import { create } from 'jsondiffpatch';
+
+const createWorkflowDiffPatcher = () =>
+  create({
+    objectHash: (obj: any) => obj.id || obj.nodeId || obj._id,
+    propertyFilter: (name: string) => name !== 'selected'
+  });
+
+const diffPatcher = createWorkflowDiffPatcher();
+
+export const getAppDiffConfig = <T extends Record<string, unknown>>(
+  initialState?: T,
+  newState?: T
+) => {
+  return diffPatcher.diff(initialState, newState);
+};
+
+export const getAppConfigByDiff = <T extends Record<string, unknown>>(
+  initialState?: T,
+  diff?: ReturnType<typeof diffPatcher.diff>
+) => {
+  return diffPatcher.patch(structuredClone(initialState), diff) as T;
+};
--- a/projects/app/src/web/core/workflow/utils.ts
+++ b/projects/app/src/web/core/workflow/utils.ts
@@ -631,3 +631,13 @@ export const compareSnapshot = (

  return isEqual(node1, node2);
 };
+
+// remove node size
+export const simplifyWorkflowNodes = (nodes: Node[]) => {
+  return nodes.map((node) => ({
+    id: node.id,
+    type: node.type,
+    position: node.position,
+    data: node.data
+  }));
+};
Author	SHA1	Message	Date
Archer	019bf67e2d	Snip test (#3204 ) * fix: index * fix: snapshot error; perf: snapshot diff compare * perf: init simple edit history	2024-11-21 16:26:43 +08:00
heheer	9b2c3b242a	refactor: snapshot store to diff (#3155 ) * refactor: snapshot store to diff * change initial state position * fix old snapshot format * encapsulate json diff	2024-11-21 13:12:42 +08:00
Archer	4f55025906	rFix textspliter (#3200 ) * fix: text splitter * perf: splitter	2024-11-21 12:01:55 +08:00
Archer	489bb076a3	更新 configuration.md (#3188 )	2024-11-19 10:19:21 +08:00
Archer	a9db5b57c5	feat: add training retry time (#3187 ) * feat: add training retry time * remoce log	2024-11-18 20:57:03 +08:00
Archer	fdb3720b41	Update 4813.md (#3185 )	2024-11-18 18:34:08 +08:00
Archer	00641a8652	Update official_account.md (#3181 )	2024-11-18 14:54:35 +08:00
Archer	5c56b375c7	Update official_account.md (#3180 )	2024-11-18 14:28:01 +08:00
Archer	d8d9b936c4	fix: custom uid (#3177 ) * chat api doc * fix: custom uid	2024-11-18 10:27:52 +08:00
Archer	b237a3ec55	更新 4813.md (#3174 )	2024-11-16 10:32:27 +08:00