From d4df77e637c7bf90bc23004ca25486385ae9d1cc Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Tue, 11 Mar 2025 21:58:24 +0800 Subject: [PATCH] perf: chunk read (#4109) * package * perf: chunk read --- .../zh-cn/docs/development/upgrading/491.md | 1 + .../global/core/dataset/data/constants.ts | 10 +- packages/service/core/chat/saveChat.ts | 65 ++++---- packages/service/core/dataset/data/schema.ts | 4 - .../common/Icon/icons/common/download.svg | 2 +- packages/web/i18n/en/chat.json | 3 + packages/web/i18n/en/common.json | 8 +- packages/web/i18n/en/dataset.json | 5 - packages/web/i18n/zh-CN/chat.json | 3 + packages/web/i18n/zh-CN/common.json | 8 +- packages/web/i18n/zh-CN/dataset.json | 5 - packages/web/i18n/zh-Hant/chat.json | 3 + packages/web/i18n/zh-Hant/common.json | 8 +- packages/web/i18n/zh-Hant/dataset.json | 5 - projects/app/package.json | 2 +- projects/app/src/components/SideBar/index.tsx | 2 +- .../app/detail/Logs/DetailLogsModal.tsx | 75 +++++----- .../app/detail/SimpleApp/ChatTest.tsx | 5 +- .../WorkflowComponents/Flow/ChatTest.tsx | 54 +++---- .../ChatQuoteList/CollectionQuoteReader.tsx | 140 +++++++----------- .../chat/ChatQuoteList/DownloadButton.tsx | 36 +---- .../chat/ChatQuoteList/ScoreTag.tsx | 6 +- .../api/core/chat/quote/getCollectionQuote.ts | 37 +++-- .../src/pages/api/core/chat/quote/getQuote.ts | 4 +- .../api/core/dataset/collection/export.ts | 11 +- projects/app/src/pages/chat/index.tsx | 108 +++++++------- projects/app/src/pages/chat/share.tsx | 127 ++++++++-------- projects/app/src/pages/chat/team.tsx | 95 ++++++------ .../service/core/dataset/data/controller.ts | 15 +- 29 files changed, 416 insertions(+), 431 deletions(-) diff --git a/docSite/content/zh-cn/docs/development/upgrading/491.md b/docSite/content/zh-cn/docs/development/upgrading/491.md index 0e0d49403..7a13176a6 100644 --- a/docSite/content/zh-cn/docs/development/upgrading/491.md +++ b/docSite/content/zh-cn/docs/development/upgrading/491.md @@ -10,6 +10,7 @@ weight: 799 ## 🚀 新增内容 1. 商业版支持单团队模式，更好的管理内部成员。 +2. 知识库分块阅读器。 ## ⚙️ 优化 diff --git a/packages/global/core/dataset/data/constants.ts b/packages/global/core/dataset/data/constants.ts index 802b5f469..7cb326fa2 100644 --- a/packages/global/core/dataset/data/constants.ts +++ b/packages/global/core/dataset/data/constants.ts @@ -16,23 +16,23 @@ export const DatasetDataIndexMap: Record< } > = { [DatasetDataIndexTypeEnum.default]: { - label: i18nT('dataset:data_index_default'), + label: i18nT('common:data_index_default'), color: 'gray' }, [DatasetDataIndexTypeEnum.custom]: { - label: i18nT('dataset:data_index_custom'), + label: i18nT('common:data_index_custom'), color: 'blue' }, [DatasetDataIndexTypeEnum.summary]: { - label: i18nT('dataset:data_index_summary'), + label: i18nT('common:data_index_summary'), color: 'green' }, [DatasetDataIndexTypeEnum.question]: { - label: i18nT('dataset:data_index_question'), + label: i18nT('common:data_index_question'), color: 'red' }, [DatasetDataIndexTypeEnum.image]: { - label: i18nT('dataset:data_index_image'), + label: i18nT('common:data_index_image'), color: 'purple' } }; diff --git a/packages/service/core/chat/saveChat.ts b/packages/service/core/chat/saveChat.ts index 94f75cd25..5a437c9a7 100644 --- a/packages/service/core/chat/saveChat.ts +++ b/packages/service/core/chat/saveChat.ts @@ -74,41 +74,42 @@ export async function saveChat({ (node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput )?.inputs; - await mongoSessionRun(async (session) => { - const processedContent = content.map((item) => { - if (item.obj === ChatRoleEnum.AI) { - const nodeResponse = item[DispatchNodeResponseKeyEnum.nodeResponse]; + // Format save chat content: Remove quote q/a + const processedContent = content.map((item) => { + if (item.obj === ChatRoleEnum.AI) { + const nodeResponse = item[DispatchNodeResponseKeyEnum.nodeResponse]; - if (nodeResponse) { - return { - ...item, - [DispatchNodeResponseKeyEnum.nodeResponse]: nodeResponse.map((responseItem) => { - if ( - responseItem.moduleType === FlowNodeTypeEnum.datasetSearchNode && - responseItem.quoteList - ) { - return { - ...item, - quoteList: responseItem.quoteList.map((quote: any) => ({ - id: quote.id, - chunkIndex: quote.chunkIndex, - datasetId: quote.datasetId, - collectionId: quote.collectionId, - sourceId: quote.sourceId, - sourceName: quote.sourceName, - score: quote.score, - tokens: quote.tokens - })) - }; - } - return item; - }) - }; - } + if (nodeResponse) { + return { + ...item, + [DispatchNodeResponseKeyEnum.nodeResponse]: nodeResponse.map((responseItem) => { + if ( + responseItem.moduleType === FlowNodeTypeEnum.datasetSearchNode && + responseItem.quoteList + ) { + return { + ...responseItem, + quoteList: responseItem.quoteList.map((quote: any) => ({ + id: quote.id, + chunkIndex: quote.chunkIndex, + datasetId: quote.datasetId, + collectionId: quote.collectionId, + sourceId: quote.sourceId, + sourceName: quote.sourceName, + score: quote.score, + tokens: quote.tokens + })) + }; + } + return responseItem; + }) + }; } - return item; - }); + } + return item; + }); + await mongoSessionRun(async (session) => { const [{ _id: chatItemIdHuman }, { _id: chatItemIdAi }] = await MongoChatItem.insertMany( processedContent.map((item) => ({ chatId, diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index 7ed3b937e..e4946d37b 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -98,15 +98,11 @@ try { chunkIndex: 1, updateTime: -1 }); - // FullText tmp full text index - // DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' }); // Recall vectors after data matching DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 }); DatasetDataSchema.index({ updateTime: 1 }); // rebuild data DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 }); - - DatasetDataSchema.index({ initFullText: 1 }); } catch (error) { console.log(error); } diff --git a/packages/web/components/common/Icon/icons/common/download.svg b/packages/web/components/common/Icon/icons/common/download.svg index f70f2fbf0..98693507a 100644 --- a/packages/web/components/common/Icon/icons/common/download.svg +++ b/packages/web/components/common/Icon/icons/common/download.svg @@ -1,3 +1,3 @@ - + \ No newline at end of file diff --git a/packages/web/i18n/en/chat.json b/packages/web/i18n/en/chat.json index 037b3d08d..b3326244f 100644 --- a/packages/web/i18n/en/chat.json +++ b/packages/web/i18n/en/chat.json @@ -20,6 +20,7 @@ "custom_input_guide_url": "Custom Lexicon URL", "dataset_quote_type error": "Knowledge base reference type is wrong, correct type: { datasetId: string }[]", "delete_all_input_guide_confirm": "Are you sure you want to clear the input guide lexicon?", + "download_chunks": "Download data", "empty_directory": "This directory is empty~", "file_amount_over": "Exceeded maximum file quantity {{max}}", "file_input": "File input", @@ -42,6 +43,7 @@ "query_extension_IO_tokens": "Problem Optimization Input/Output Tokens", "query_extension_result": "Problem optimization results", "question_tip": "From top to bottom, the response order of each module", + "read_raw_source": "Open the original text", "reasoning_text": "Thinking process", "response.child total points": "Sub-workflow point consumption", "response.dataset_concat_length": "Combined total", @@ -52,6 +54,7 @@ "select_img": "Upload Image", "source_cronJob": "Scheduled execution", "stream_output": "Stream Output", + "to_dataset": "Go to the Knowledge Base", "unsupported_file_type": "Unsupported file types", "upload": "Upload", "view_citations": "View References", diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index e81162a04..d42335a01 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -498,12 +498,9 @@ "core.dataset.Dataset": "Dataset", "core.dataset.Dataset ID": "Dataset ID", "core.dataset.Delete Confirm": "Confirm to Delete This Dataset? Data Cannot Be Recovered After Deletion, Please Confirm!", - "core.dataset.Download the parsed content": "Download the parsed content", "core.dataset.Empty Dataset": "Empty Dataset", "core.dataset.Empty Dataset Tips": "No Dataset Yet, Create One Now!", "core.dataset.Folder placeholder": "This is a Directory", - "core.dataset.Get the raw data": "Get the raw data", - "core.dataset.Go Dataset": "Go to Dataset", "core.dataset.Intro Placeholder": "This Dataset Has No Introduction Yet", "core.dataset.Manual collection": "Manual Dataset", "core.dataset.My Dataset": "My Dataset", @@ -832,6 +829,11 @@ "core.workflow.variable": "Variable", "create": "Create", "cron_job_run_app": "Scheduled Task", + "data_index_custom": "Custom index", + "data_index_default": "Default index", + "data_index_image": "Image Index", + "data_index_question": "Inferred question index", + "data_index_summary": "Summary Index", "dataset.Confirm move the folder": "Confirm to Move to This Directory", "dataset.Confirm to delete the data": "Confirm to Delete This Data?", "dataset.Confirm to delete the file": "Confirm to Delete This File and All Its Data?", diff --git a/packages/web/i18n/en/dataset.json b/packages/web/i18n/en/dataset.json index 0b9a951c8..7a57c3e97 100644 --- a/packages/web/i18n/en/dataset.json +++ b/packages/web/i18n/en/dataset.json @@ -27,12 +27,7 @@ "custom_data_process_params_desc": "Customize data processing rules", "data.ideal_chunk_length": "ideal block length", "data_amount": "{{dataAmount}} Datas, {{indexAmount}} Indexes", - "data_index_custom": "Custom index", - "data_index_default": "Default index", - "data_index_image": "Image Index", "data_index_num": "Index {{index}}", - "data_index_question": "Inferred question index", - "data_index_summary": "Summary index", "data_process_params": "Params", "data_process_setting": "Processing config", "dataset.Unsupported operation": "dataset.Unsupported operation", diff --git a/packages/web/i18n/zh-CN/chat.json b/packages/web/i18n/zh-CN/chat.json index a8cefa2db..4f8d9e476 100644 --- a/packages/web/i18n/zh-CN/chat.json +++ b/packages/web/i18n/zh-CN/chat.json @@ -20,6 +20,7 @@ "custom_input_guide_url": "自定义词库地址", "dataset_quote_type error": "知识库引用类型错误，正确类型：{ datasetId: string }[]", "delete_all_input_guide_confirm": "确定要清空输入引导词库吗？", + "download_chunks": "下载数据", "empty_directory": "这个目录已经没东西可选了~", "file_amount_over": "超出最大文件数量 {{max}}", "file_input": "系统文件", @@ -42,6 +43,7 @@ "query_extension_IO_tokens": "问题优化输入/输出 Tokens", "query_extension_result": "问题优化结果", "question_tip": "从上到下，为各个模块的响应顺序", + "read_raw_source": "打开原文", "reasoning_text": "思考过程", "response.child total points": "子工作流积分消耗", "response.dataset_concat_length": "合并后总数", @@ -52,6 +54,7 @@ "select_img": "上传图片", "source_cronJob": "定时执行", "stream_output": "流输出", + "to_dataset": "前往知识库", "unsupported_file_type": "不支持的文件类型", "upload": "上传", "view_citations": "查看引用", diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index 8eaed9e0a..5317f35f0 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -501,12 +501,9 @@ "core.dataset.Dataset": "知识库", "core.dataset.Dataset ID": "知识库 ID", "core.dataset.Delete Confirm": "确认删除该知识库？删除后数据无法恢复，请确认！", - "core.dataset.Download the parsed content": "下载解析内容", "core.dataset.Empty Dataset": "空数据集", "core.dataset.Empty Dataset Tips": "还没有知识库，快去创建一个吧！", "core.dataset.Folder placeholder": "这是一个目录", - "core.dataset.Get the raw data": "获取源数据", - "core.dataset.Go Dataset": "前往知识库", "core.dataset.Intro Placeholder": "这个知识库还没有介绍~", "core.dataset.Manual collection": "手动数据集", "core.dataset.My Dataset": "我的知识库", @@ -836,6 +833,11 @@ "core.workflow.variable": "变量", "create": "去创建", "cron_job_run_app": "定时任务", + "data_index_custom": "自定义索引", + "data_index_default": "默认索引", + "data_index_image": "图片索引", + "data_index_question": "推测问题索引", + "data_index_summary": "摘要索引", "dataset.Confirm move the folder": "确认移动到该目录", "dataset.Confirm to delete the data": "确认删除该数据？", "dataset.Confirm to delete the file": "确认删除该文件及其所有数据？", diff --git a/packages/web/i18n/zh-CN/dataset.json b/packages/web/i18n/zh-CN/dataset.json index 064b5a73b..7dd79ee32 100644 --- a/packages/web/i18n/zh-CN/dataset.json +++ b/packages/web/i18n/zh-CN/dataset.json @@ -27,12 +27,7 @@ "custom_data_process_params_desc": "自定义设置数据处理规则", "data.ideal_chunk_length": "理想分块长度", "data_amount": "{{dataAmount}} 组数据, {{indexAmount}} 组索引", - "data_index_custom": "自定义索引", - "data_index_default": "默认索引", - "data_index_image": "图片索引", "data_index_num": "索引 {{index}}", - "data_index_question": "推测问题索引", - "data_index_summary": "摘要索引", "data_process_params": "处理参数", "data_process_setting": "数据处理配置", "dataset.Unsupported operation": "操作不支持", diff --git a/packages/web/i18n/zh-Hant/chat.json b/packages/web/i18n/zh-Hant/chat.json index b7494de5f..bceac9809 100644 --- a/packages/web/i18n/zh-Hant/chat.json +++ b/packages/web/i18n/zh-Hant/chat.json @@ -20,6 +20,7 @@ "custom_input_guide_url": "自訂詞彙庫網址", "dataset_quote_type error": "知識庫引用類型錯誤，正確類型：{ datasetId: string }[]", "delete_all_input_guide_confirm": "確定要清除輸入導引詞彙庫嗎？", + "download_chunks": "下載數據", "empty_directory": "此目錄中已無項目可選～", "file_amount_over": "超出檔案數量上限 {{max}}", "file_input": "檔案輸入", @@ -41,6 +42,7 @@ "plugins_output": "外掛程式輸出", "query_extension_IO_tokens": "問題優化輸入/輸出 Tokens", "question_tip": "由上至下，各個模組的回應順序", + "read_raw_source": "打開原文", "reasoning_text": "思考過程", "response.child total points": "子工作流程點數消耗", "response.dataset_concat_length": "合併總數", @@ -51,6 +53,7 @@ "select_img": "上傳圖片", "source_cronJob": "定時執行", "stream_output": "串流輸出", + "to_dataset": "前往知識庫", "unsupported_file_type": "不支援的檔案類型", "upload": "上傳", "view_citations": "檢視引用", diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index c6a3771c2..63da238a9 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -497,12 +497,9 @@ "core.dataset.Dataset": "知識庫", "core.dataset.Dataset ID": "知識庫 ID", "core.dataset.Delete Confirm": "確認刪除此知識庫？刪除後資料無法復原，請確認！", - "core.dataset.Download the parsed content": "下載解析內容", "core.dataset.Empty Dataset": "空資料集", "core.dataset.Empty Dataset Tips": "還沒有知識庫，快來建立一個吧！", "core.dataset.Folder placeholder": "這是一個目錄", - "core.dataset.Get the raw data": "獲取源數據", - "core.dataset.Go Dataset": "前往知識庫", "core.dataset.Intro Placeholder": "這個知識庫還沒有介紹", "core.dataset.Manual collection": "手動資料集", "core.dataset.My Dataset": "我的知識庫", @@ -831,6 +828,11 @@ "core.workflow.variable": "變數", "create": "建立", "cron_job_run_app": "排程任務", + "data_index_custom": "自定義索引", + "data_index_default": "默認索引", + "data_index_image": "圖片索引", + "data_index_question": "推測問題索引", + "data_index_summary": "摘要索引", "dataset.Confirm move the folder": "確認移動到此目錄", "dataset.Confirm to delete the data": "確認刪除此資料？", "dataset.Confirm to delete the file": "確認刪除此檔案及其所有資料？", diff --git a/packages/web/i18n/zh-Hant/dataset.json b/packages/web/i18n/zh-Hant/dataset.json index 706714283..712956760 100644 --- a/packages/web/i18n/zh-Hant/dataset.json +++ b/packages/web/i18n/zh-Hant/dataset.json @@ -27,12 +27,7 @@ "custom_data_process_params_desc": "自訂資料處理規則", "data.ideal_chunk_length": "理想分塊長度", "data_amount": "{{dataAmount}} 組數據, {{indexAmount}} 組索引", - "data_index_custom": "自定義索引", - "data_index_default": "默認索引", - "data_index_image": "圖片索引", "data_index_num": "索引 {{index}}", - "data_index_question": "推測問題索引", - "data_index_summary": "摘要索引", "data_process_params": "處理參數", "data_process_setting": "資料處理設定", "dataset.Unsupported operation": "操作不支持", diff --git a/projects/app/package.json b/projects/app/package.json index 354ce0a2e..f37722b66 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -1,6 +1,6 @@ { "name": "app", - "version": "4.9.0", + "version": "4.9.1", "private": false, "scripts": { "dev": "next dev", diff --git a/projects/app/src/components/SideBar/index.tsx b/projects/app/src/components/SideBar/index.tsx index 27bd64861..3af058e14 100644 --- a/projects/app/src/components/SideBar/index.tsx +++ b/projects/app/src/components/SideBar/index.tsx @@ -9,7 +9,7 @@ interface Props extends BoxProps { const SideBar = (e?: Props) => { const { - w = ['100%', '0 0 250px', '0 0 270px', '0 0 290px', '0 0 310px'], + w = ['100%', '0 0 250px', '0 0 250px', '0 0 270px', '0 0 290px'], children, externalTrigger, ...props diff --git a/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx b/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx index 32cdebdf6..0606a33b7 100644 --- a/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx +++ b/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx @@ -79,10 +79,10 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => { zIndex={3} position={['fixed', 'absolute']} top={[0, '2%']} - right={quoteData ? 600 : 0} + right={0} h={['100%', '96%']} w={'100%'} - maxW={['100%', '600px']} + maxW={quoteData ? ['100%', '1080px'] : ['100%', '600px']} bg={'white'} boxShadow={'3px 0 20px rgba(0,0,0,0.2)'} borderRadius={'md'} @@ -151,46 +151,49 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => { )} {/* Chat container */} - - {isPlugin ? ( - + + + {isPlugin ? ( + ) : ( + + )} + + + {quoteData && ( + + setQuoteData(undefined)} + /> - ) : ( - )} - + - {quoteData && ( - - setQuoteData(undefined)} - /> - - )} + ); diff --git a/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx b/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx index ed38f89b3..e020b35c4 100644 --- a/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx +++ b/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx @@ -51,12 +51,13 @@ const ChatTest = ({ appForm, setRenderEdit }: Props) => { return ( { {quoteData && ( - + { flexDirection={'column'} position={'absolute'} top={5} - right={quoteData ? 600 : 0} + right={0} h={isOpen ? '95%' : '0'} - w={isOpen ? ['100%', '460px'] : '0'} + w={isOpen ? (quoteData ? ['100%', '960px'] : ['100%', '460px']) : '0'} bg={'white'} boxShadow={'3px 0 20px rgba(0,0,0,0.2)'} borderRadius={'md'} @@ -144,30 +144,34 @@ const ChatTest = ({ isOpen, nodes = [], edges = [], onClose }: Props) => { )} - - - + + + + + + {quoteData && ( + + setQuoteData(undefined)} + /> + + )} + - {quoteData && ( - - setQuoteData(undefined)} - /> - - )} ); }; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx index 53dffec23..698349259 100644 --- a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx @@ -1,4 +1,4 @@ -import { Box, Button, Flex } from '@chakra-ui/react'; +import { Box, Button, Flex, HStack } from '@chakra-ui/react'; import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; import MyIcon from '@fastgpt/web/components/common/Icon'; @@ -21,6 +21,10 @@ import { DatasetDataListItemType } from '@/global/core/dataset/type'; import { metadataType } from '@/web/core/chat/context/chatItemContext'; import { useUserStore } from '@/web/support/user/useUserStore'; import { getCollectionQuote } from '@/web/core/chat/api'; +import MyIconButton from '@fastgpt/web/components/common/Icon/button'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; +import MyBox from '@fastgpt/web/components/common/MyBox'; +import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource'; const CollectionReader = ({ rawSearch, @@ -105,10 +109,10 @@ const CollectionReader = ({ setDatasetDataList([]); }, [collectionId, setDatasetDataList]); - const { runAsync: handleDownload, loading: downloadLoading } = useRequest2(async () => { + const { runAsync: handleDownload } = useRequest2(async () => { await downloadFetch({ url: '/api/core/dataset/collection/export', - filename: 'parsed_content.md', + filename: 'data.csv', body: { collectionId: collectionId, chatTime: chatTime, @@ -117,27 +121,11 @@ const CollectionReader = ({ }); }); - const { runAsync: handleRead, loading: readLoading } = useRequest2( - async () => await getCollectionSource({ ...metadata, appId, chatId }), - { - onSuccess: (res) => { - if (!res.value) { - throw new Error('No file found'); - } - if (res.value.startsWith('/')) { - window.open(`${location.origin}${res.value}`, '_blank'); - } else { - window.open(res.value, '_blank'); - } - }, - onError: (err) => { - toast({ - title: t(getErrText(err, t('common:error.fileNotFound'))), - status: 'error' - }); - } - } - ); + const handleRead = getCollectionSourceAndOpen({ + appId, + chatId, + ...metadata + }); const handleNavigate = useCallback( async (targetIndex: number) => { @@ -164,77 +152,61 @@ const CollectionReader = ({ ); return ( - + {/* title */} - - - - - - - {sourceName || t('common:common.UnKnow Source')} - - - - {!!userInfo && permissionData?.permission?.hasReadPer && ( - - )} + /> + + )} + - + - - {t('common:core.chat.quote.Quote Tip')} - + + + + {t('common:core.chat.quote.Quote Tip')} - - - - + {/* header control */} {datasetDataList.length > 0 && ( @@ -299,7 +271,7 @@ const CollectionReader = ({ {/* quote list */} {loading || datasetDataList.length > 0 ? ( - + {formatedDataList.map((item, index) => ( )} - + ); }; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/DownloadButton.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/DownloadButton.tsx index 10fa34424..0dd3e8b8f 100644 --- a/projects/app/src/pageComponents/chat/ChatQuoteList/DownloadButton.tsx +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/DownloadButton.tsx @@ -1,18 +1,15 @@ -import { Button } from '@chakra-ui/react'; import MyMenu from '@fastgpt/web/components/common/MyMenu'; import { useTranslation } from 'react-i18next'; -import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyIconButton from '@fastgpt/web/components/common/Icon/button'; const DownloadButton = ({ canAccessRawData, onDownload, - onRead, - isLoading + onRead }: { canAccessRawData: boolean; onDownload: () => void; onRead: () => void; - isLoading: boolean; }) => { const { t } = useTranslation(); @@ -20,27 +17,17 @@ const DownloadButton = ({ return ( } - isLoading={isLoading} - > - {t('common:Download')} - - } + Button={} menuList={[ { children: [ { - label: t('common:core.dataset.Download the parsed content'), + label: t('chat:download_chunks'), type: 'grayBg', onClick: onDownload }, { - label: t('common:core.dataset.Get the raw data'), + label: t('chat:read_raw_source'), type: 'grayBg', onClick: onRead } @@ -51,18 +38,7 @@ const DownloadButton = ({ ); } - return ( - } - onClick={onDownload} - isLoading={isLoading} - > - {t('common:Download')} - - ); + return ; }; export default DownloadButton; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/ScoreTag.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/ScoreTag.tsx index 16b8a00d5..702992d59 100644 --- a/projects/app/src/pageComponents/chat/ChatQuoteList/ScoreTag.tsx +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/ScoreTag.tsx @@ -13,9 +13,9 @@ const ScoreTag = (score: { primaryScore?: ScoreItemType; secondaryScore: ScoreIt + {score.secondaryScore.map((item, i) => ( - + ))} - + ) : ( t(SearchScoreTypeMap[score.primaryScore.type]?.desc as any) ) diff --git a/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts b/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts index 99ffc0d51..1f99ac26b 100644 --- a/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts +++ b/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts @@ -6,6 +6,8 @@ import { ApiRequestProps } from '@fastgpt/service/type/next'; import { LinkedListResponse, LinkedPaginationProps } from '@fastgpt/web/common/fetch/type'; import { FilterQuery, Types } from 'mongoose'; import { dataFieldSelector, processChatTimeFilter } from './getQuote'; +import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; +import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; export type GetCollectionQuoteProps = LinkedPaginationProps & { chatTime: Date; @@ -36,8 +38,8 @@ async function handler( nextId, nextIndex, chatTime, - isInitialLoad, + collectionId, chatItemId, appId, @@ -48,21 +50,32 @@ async function handler( teamToken, pageSize = 15 } = req.body; + const limitedPageSize = Math.min(pageSize, 30); - await Promise.all([ - authChatCrud({ + try { + await authDatasetCollection({ req, authToken: true, - appId, - chatId, - shareId, - outLinkUid, - teamId, - teamToken - }), - authCollectionInChat({ appId, chatId, chatItemId, collectionId }) - ]); + authApiKey: true, + collectionId: req.body.collectionId, + per: ReadPermissionVal + }); + } catch (error) { + await Promise.all([ + authChatCrud({ + req, + authToken: true, + appId, + chatId, + shareId, + outLinkUid, + teamId, + teamToken + }), + authCollectionInChat({ appId, chatId, chatItemId, collectionId }) + ]); + } const baseMatch: BaseMatchType = { collectionId, diff --git a/projects/app/src/pages/api/core/chat/quote/getQuote.ts b/projects/app/src/pages/api/core/chat/quote/getQuote.ts index c6e35f96f..705ef1c78 100644 --- a/projects/app/src/pages/api/core/chat/quote/getQuote.ts +++ b/projects/app/src/pages/api/core/chat/quote/getQuote.ts @@ -71,9 +71,7 @@ async function handler(req: ApiRequestProps): Promise { if (!item.history) return item; diff --git a/projects/app/src/pages/api/core/dataset/collection/export.ts b/projects/app/src/pages/api/core/dataset/collection/export.ts index d9111e806..2ad805b2e 100644 --- a/projects/app/src/pages/api/core/dataset/collection/export.ts +++ b/projects/app/src/pages/api/core/dataset/collection/export.ts @@ -39,7 +39,7 @@ async function handler(req: ApiRequestProps, res: Next }; res.setHeader('Content-Type', 'text/csv; charset=utf-8;'); - res.setHeader('Content-Disposition', 'attachment; filename=usage.csv; '); + res.setHeader('Content-Disposition', 'attachment; filename=data.csv; '); const cursor = MongoDatasetData.find(where, 'q a', { ...readFromSecondary, @@ -54,10 +54,13 @@ async function handler(req: ApiRequestProps, res: Next readStream: cursor }); - cursor.on('data', (doc) => { - const res = doc.a ? `\n${doc.q}\n${doc.a}` : `\n${doc.q}`; + write(`\uFEFFindex,content`); - write(res); + cursor.on('data', (doc) => { + const q = doc.q.replace(/"/g, '""') || ''; + const a = doc.a.replace(/"/g, '""') || ''; + + write(`\n"${q}","${a}"`); }); cursor.on('end', () => { diff --git a/projects/app/src/pages/chat/index.tsx b/projects/app/src/pages/chat/index.tsx index 0802e50d0..6e2c7399f 100644 --- a/projects/app/src/pages/chat/index.tsx +++ b/projects/app/src/pages/chat/index.tsx @@ -168,66 +168,68 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => { {/* pc show myself apps */} {isPc && ( - + )} - - - {/* pc always show history. */} - {RenderHistorySlider} - {/* chat container */} - - {/* header */} - + {(!quoteData || isPc) && ( + + + {/* pc always show history. */} + {RenderHistorySlider} + {/* chat container */} + + {/* header */} + - {/* chat box */} - - {isPlugin ? ( - onChangeChatId(getNanoid())} - onStartChat={onStartChat} - /> - ) : ( - - )} - + {/* chat box */} + + {isPlugin ? ( + onChangeChatId(getNanoid())} + onStartChat={onStartChat} + /> + ) : ( + + )} + + - - + + )} + {quoteData && ( - + { }, [isOpenSlider, isPc, onCloseSlider, quoteData, showHistory, t]); return ( - + <> - - - {RenderHistoryList} - - {/* chat container */} - + {(!quoteData || isPc) && ( + - {/* header */} - {showHead === '1' ? ( - - ) : null} - {/* chat box */} - - {isPlugin ? ( - onChangeChatId(getNanoid())} - onStartChat={startChat} - /> - ) : ( - - )} - - - - - {quoteData && ( - - setQuoteData(undefined)} - /> - - )} - + + {RenderHistoryList} + + {/* chat container */} + + {/* header */} + {showHead === '1' ? ( + + ) : null} + {/* chat box */} + + {isPlugin ? ( + onChangeChatId(getNanoid())} + onStartChat={startChat} + /> + ) : ( + + )} + + + + + )} + + {quoteData && ( + + setQuoteData(undefined)} + /> + + )} + + ); }; diff --git a/projects/app/src/pages/chat/team.tsx b/projects/app/src/pages/chat/team.tsx index dd02531e7..e45927f1d 100644 --- a/projects/app/src/pages/chat/team.tsx +++ b/projects/app/src/pages/chat/team.tsx @@ -191,52 +191,59 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => { )} - - - {RenderHistoryList} - {/* chat container */} - - {/* header */} - - {/* chat box */} - - {chatBoxData.app.type === AppTypeEnum.plugin ? ( - onChangeChatId(getNanoid())} - onStartChat={startChat} - /> - ) : ( - - )} - + {(!quoteData || isPc) && ( + + + {RenderHistoryList} + {/* chat container */} + + {/* header */} + + {/* chat box */} + + {chatBoxData.app.type === AppTypeEnum.plugin ? ( + onChangeChatId(getNanoid())} + onStartChat={startChat} + /> + ) : ( + + )} + + - - - + + )} {quoteData && ( - + item.type === 'create' || item.type === 'update') @@ -245,15 +249,16 @@ export async function updateData2Dataset({ .filter((item) => item.type !== 'delete') .map((item) => item.index) as DatasetDataIndexItemType[]; + // 6. update mongo data await mongoSessionRun(async (session) => { - // update mongo data + // Update history mongoData.history = q !== mongoData.q || a !== mongoData.a ? [ { q: mongoData.q, a: mongoData.a, - updateTime: mongoData.updateTime + updateTime: new Date() }, ...(mongoData.history?.slice(0, 9) || []) ] @@ -283,10 +288,6 @@ export async function updateData2Dataset({ } }); - // Update mongo updateTime(便于脏数据检查器识别) - mongoData.updateTime = new Date(); - await mongoData.save(); - return { tokens };