Add tests for getFlatAppResponses function in chat utils.

docSite/content/zh-cn/docs/development/upgrading/4910.md

@@ -22,13 +22,10 @@ weight: 790
 3. 纠正原先知识库的“表格数据集”名称，改成“备份导入”。同时支持知识库索引的导出和导入。
 4. 工作流知识库引用上限，如果工作流中没有相关 AI 节点，则交互模式改成纯手动输入，并且上限为 1000万。
 5. 语音输入，移动端判断逻辑，准确判断是否为手机，而不是小屏。
-6. 优化上下文截取算法，至少保证留下一组 Human 信息。
 
 ## 🐛 修复
 
 1. 全文检索多知识库时排序得分排序不正确。
 2. 流响应捕获 finish_reason 可能不正确。
 3. 工具调用模式，未保存思考输出。
-4. 知识库 indexSize 参数未生效。
+4. 知识库 indexSize 参数未生效。
-5. 工作流嵌套 2 层后，获取预览引用、上下文不正确。
-6. xlsx 转成 Markdown 时候，前面会多出一个空格。

docSite/content/zh-cn/docs/shopping_cart/intro.md

@@ -28,6 +28,7 @@ FastGPT 商业版是基于 FastGPT 开源版的增强版本，增加了一些独
 | 应用发布安全配置 | ❌ | ✅ | ✅ |
 | 内容审核 | ❌ | ✅ | ✅ |
 | web站点同步 | ❌ | ✅ | ✅ |
+| 主流文档库接入（目前支持：语雀、飞书） | ❌ | ✅ | ✅ |
 | 增强训练模式 | ❌ | ✅ | ✅ |
 | 第三方应用快速接入（飞书、公众号） | ❌ | ✅ | ✅ |
 | 管理后台 | ❌ | ✅ | 不需要 |

packages/service/core/chat/utils.ts

@@ -65,8 +65,8 @@ export const filterGPTMessageByMaxContext = async ({
     if (lastMessage.role === ChatCompletionRequestMessageRoleEnum.User) {
       const tokens = await countGptMessagesTokens([lastMessage, ...tmpChats]);
       maxContext -= tokens;
-      // 该轮信息整体 tokens 超出范围，这段数据不要了。但是至少保证一组。
+      // 该轮信息整体 tokens 超出范围，这段数据不要了
-      if (maxContext < 0 && chats.length > 0) {
+      if (maxContext < 0) {
         break;
       }
 

packages/service/worker/readFile/extension/xlsx.ts

@@ -28,11 +28,11 @@ export const readXlsxRawText = async ({
       if (!header) return;
 
       const formatText = `| ${header.join(' | ')} |
-| ${header.map(() => '---').join(' | ')} |
+      | ${header.map(() => '---').join(' | ')} |
-${csvArr
+      ${csvArr
-  .slice(1)
+        .slice(1)
-  .map((row) => `| ${row.map((item) => item.replace(/\n/g, '\\n')).join(' | ')} |`)
+        .map((row) => `| ${row.map((item) => item.replace(/\n/g, '\\n')).join(' | ')} |`)
-  .join('\n')}`;
+        .join('\n')}`;
 
       return formatText;
     })

projects/app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app",
-  "version": "4.9.10",
+  "version": "4.9.9",
   "private": false,
   "scripts": {
     "dev": "next dev",

projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ContextModal.tsx

@@ -7,7 +7,6 @@ import { type ChatHistoryItemResType } from '@fastgpt/global/core/chat/type';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { useTranslation } from 'next-i18next';
-import { getFlatAppResponses } from '@/global/core/chat/utils';
 const isLLMNode = (item: ChatHistoryItemResType) =>
   item.moduleType === FlowNodeTypeEnum.chatNode || item.moduleType === FlowNodeTypeEnum.tools;
 
@@ -17,7 +16,17 @@ const ContextModal = ({ onClose, dataId }: { onClose: () => void; dataId: string
   const { loading: isLoading, data: contextModalData } = useRequest2(
     () =>
       getHistoryResponseData({ dataId }).then((res) => {
-        const flatResData = getFlatAppResponses(res || []);
+        const flatResData: ChatHistoryItemResType[] =
+          res
+            ?.map((item) => {
+              return [
+                item,
+                ...(item.pluginDetail || []),
+                ...(item.toolDetail || []),
+                ...(item.loopDetail || [])
+              ];
+            })
+            .flat() || [];
         return flatResData.find(isLLMNode)?.historyPreview || [];
       }),
     { manual: false }

projects/app/src/global/core/chat/utils.ts

@@ -19,25 +19,23 @@ export function transformPreviewHistories(
   });
 }
 
-export const getFlatAppResponses = (res: ChatHistoryItemResType[]): ChatHistoryItemResType[] => {
-  return res
-    .map((item) => {
-      return [
-        item,
-        ...getFlatAppResponses(item.pluginDetail || []),
-        ...getFlatAppResponses(item.toolDetail || []),
-        ...getFlatAppResponses(item.loopDetail || [])
-      ];
-    })
-    .flat();
-};
 export function addStatisticalDataToHistoryItem(historyItem: ChatItemType) {
   if (historyItem.obj !== ChatRoleEnum.AI) return historyItem;
   if (historyItem.totalQuoteList !== undefined) return historyItem;
   if (!historyItem.responseData) return historyItem;
 
   // Flat children
-  const flatResData = getFlatAppResponses(historyItem.responseData || []);
+  const flatResData: ChatHistoryItemResType[] =
+    historyItem.responseData
+      ?.map((item) => {
+        return [
+          item,
+          ...(item.pluginDetail || []),
+          ...(item.toolDetail || []),
+          ...(item.loopDetail || [])
+        ];
+      })
+      .flat() || [];
 
   return {
     ...historyItem,

projects/app/src/pages/api/core/dataset/collection/create/backup.ts

@@ -48,7 +48,7 @@ async function handler(req: ApiRequestProps<backupBody, backupQuery>, res: ApiRe
       encoding: file.encoding,
       getFormatText: false
     });
-    if (!rawText.trim().startsWith('q,a,indexes')) {
+    if (!rawText.startsWith('q,a,indexes')) {
       return Promise.reject('Backup file start with "q,a,indexes"');
     }
     // 2. delete tmp file

projects/app/src/pages/api/core/dataset/exportAll.ts

@@ -50,10 +50,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   });
 
   res.setHeader('Content-Type', 'text/csv; charset=utf-8;');
-  res.setHeader(
+  res.setHeader('Content-Disposition', `attachment; filename=${dataset.name}-backup.csv;`);
-    'Content-Disposition',
-    `attachment; filename=${encodeURIComponent(dataset.name)}-backup.csv;`
-  );
 
   const cursor = MongoDatasetData.find<DataItemType>(
     {

projects/app/src/service/support/permission/auth/chat.ts

@@ -1,4 +1,8 @@
-import { type ChatHistoryItemResType, type ChatSchema } from '@fastgpt/global/core/chat/type';
+import {
+  type AIChatItemType,
+  type ChatHistoryItemResType,
+  type ChatSchema
+} from '@fastgpt/global/core/chat/type';
 import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
 import { type AuthModeType } from '@fastgpt/service/support/permission/type';
 import { authOutLink } from './outLink';
@@ -8,7 +12,6 @@ import { AuthUserTypeEnum, ReadPermissionVal } from '@fastgpt/global/support/per
 import { authApp } from '@fastgpt/service/support/permission/app/auth';
 import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
 import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
-import { getFlatAppResponses } from '@/global/core/chat/utils';
 
 /* 
   检查chat的权限：
@@ -218,7 +221,18 @@ export const authCollectionInChat = async ({
     if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
 
     // 找 responseData 里，是否有该文档 id
-    const flatResData = getFlatAppResponses(chatItem.responseData || []);
+    const responseData = chatItem.responseData || [];
+    const flatResData: ChatHistoryItemResType[] =
+      responseData
+        ?.map((item) => {
+          return [
+            item,
+            ...(item.pluginDetail || []),
+            ...(item.toolDetail || []),
+            ...(item.loopDetail || [])
+          ];
+        })
+        .flat() || [];
 
     const quoteListSet = new Set(
       flatResData

test/cases/global/core/chat/utils.test.ts

@@ -4,7 +4,8 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 import type { ChatItemType } from '@fastgpt/global/core/chat/type';
 import {
   transformPreviewHistories,
-  addStatisticalDataToHistoryItem
+  addStatisticalDataToHistoryItem,
+  getFlatAppResponses
 } from '@/global/core/chat/utils';
 
 const mockResponseData = {
@@ -14,6 +15,70 @@ const mockResponseData = {
   moduleType: FlowNodeTypeEnum.chatNode
 };
 
+describe('getFlatAppResponses', () => {
+  it('should return empty array for empty input', () => {
+    expect(getFlatAppResponses([])).toEqual([]);
+  });
+
+  it('should handle single level responses', () => {
+    const responses = [
+      { ...mockResponseData, moduleType: FlowNodeTypeEnum.chatNode },
+      { ...mockResponseData, moduleType: FlowNodeTypeEnum.tools }
+    ];
+    expect(getFlatAppResponses(responses)).toEqual(responses);
+  });
+
+  it('should handle nested pluginDetail', () => {
+    const responses = [
+      {
+        ...mockResponseData,
+        pluginDetail: [{ ...mockResponseData, moduleType: FlowNodeTypeEnum.tools }]
+      }
+    ];
+    expect(getFlatAppResponses(responses)).toHaveLength(2);
+  });
+
+  it('should handle nested toolDetail', () => {
+    const responses = [
+      {
+        ...mockResponseData,
+        toolDetail: [{ ...mockResponseData, moduleType: FlowNodeTypeEnum.chatNode }]
+      }
+    ];
+    expect(getFlatAppResponses(responses)).toHaveLength(2);
+  });
+
+  it('should handle nested loopDetail', () => {
+    const responses = [
+      {
+        ...mockResponseData,
+        loopDetail: [{ ...mockResponseData, moduleType: FlowNodeTypeEnum.datasetSearchNode }]
+      }
+    ];
+    expect(getFlatAppResponses(responses)).toHaveLength(2);
+  });
+
+  it('should handle multiple levels of nesting', () => {
+    const responses = [
+      {
+        ...mockResponseData,
+        pluginDetail: [
+          {
+            ...mockResponseData,
+            toolDetail: [
+              {
+                ...mockResponseData,
+                loopDetail: [{ ...mockResponseData }]
+              }
+            ]
+          }
+        ]
+      }
+    ];
+    expect(getFlatAppResponses(responses)).toHaveLength(4);
+  });
+});
+
 describe('transformPreviewHistories', () => {
   it('should transform histories correctly with responseDetail=true', () => {
     const histories: ChatItemType[] = [