diff --git a/packages/global/core/chat/type.d.ts b/packages/global/core/chat/type.d.ts index 8837075e8..8594a0c16 100644 --- a/packages/global/core/chat/type.d.ts +++ b/packages/global/core/chat/type.d.ts @@ -134,6 +134,7 @@ export type ChatItemType = (UserChatItemType | SystemChatItemType | AIChatItemTy // Frontend type export type ChatSiteItemType = (UserChatItemType | SystemChatItemType | AIChatItemType) & { + _id?: string; dataId: string; status: `${ChatStatusEnum}`; moduleName?: string; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 74741a2a9..c9b0aa899 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -112,12 +112,15 @@ export type DatasetDataSchemaType = { tmbId: string; datasetId: string; collectionId: string; - datasetId: string; - collectionId: string; chunkIndex: number; updateTime: Date; q: string; // large chunks or question a: string; // answer or custom content + history?: { + q: string; + a: string; + updateTime: Date; + }[]; forbid?: boolean; fullTextToken: string; indexes: DatasetDataIndexItemType[]; diff --git a/packages/global/support/outLink/type.d.ts b/packages/global/support/outLink/type.d.ts index b1c74e60c..b3e1ef082 100644 --- a/packages/global/support/outLink/type.d.ts +++ b/packages/global/support/outLink/type.d.ts @@ -63,6 +63,8 @@ export type OutLinkSchema = { responseDetail: boolean; // whether to hide the node status showNodeStatus?: boolean; + // wheter to show the full text reader + // showFullText?: boolean; // whether to show the complete quote showRawSource?: boolean; @@ -89,6 +91,7 @@ export type OutLinkEditType = { name: string; responseDetail?: OutLinkSchema['responseDetail']; showNodeStatus?: OutLinkSchema['showNodeStatus']; + // showFullText?: OutLinkSchema['showFullText']; showRawSource?: OutLinkSchema['showRawSource']; // response when request immediateResponse?: string; diff --git a/packages/service/core/chat/saveChat.ts b/packages/service/core/chat/saveChat.ts index 48bbcc8f4..94f75cd25 100644 --- a/packages/service/core/chat/saveChat.ts +++ b/packages/service/core/chat/saveChat.ts @@ -15,6 +15,7 @@ import { AppChatConfigType } from '@fastgpt/global/core/app/type'; import { mergeChatResponseData } from '@fastgpt/global/core/chat/utils'; import { pushChatLog } from './pushChatLog'; import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; +import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants'; type Props = { chatId: string; @@ -74,8 +75,42 @@ export async function saveChat({ )?.inputs; await mongoSessionRun(async (session) => { + const processedContent = content.map((item) => { + if (item.obj === ChatRoleEnum.AI) { + const nodeResponse = item[DispatchNodeResponseKeyEnum.nodeResponse]; + + if (nodeResponse) { + return { + ...item, + [DispatchNodeResponseKeyEnum.nodeResponse]: nodeResponse.map((responseItem) => { + if ( + responseItem.moduleType === FlowNodeTypeEnum.datasetSearchNode && + responseItem.quoteList + ) { + return { + ...item, + quoteList: responseItem.quoteList.map((quote: any) => ({ + id: quote.id, + chunkIndex: quote.chunkIndex, + datasetId: quote.datasetId, + collectionId: quote.collectionId, + sourceId: quote.sourceId, + sourceName: quote.sourceName, + score: quote.score, + tokens: quote.tokens + })) + }; + } + return item; + }) + }; + } + } + return item; + }); + const [{ _id: chatItemIdHuman }, { _id: chatItemIdAi }] = await MongoChatItem.insertMany( - content.map((item) => ({ + processedContent.map((item) => ({ chatId, teamId, tmbId, diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index bdba3b87c..7ed3b937e 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -40,6 +40,15 @@ const DatasetDataSchema = new Schema({ type: String, default: '' }, + history: { + type: [ + { + q: String, + a: String, + updateTime: Date + } + ] + }, indexes: { type: [ { diff --git a/packages/service/support/outLink/schema.ts b/packages/service/support/outLink/schema.ts index 0c6d5ea4c..0752e28b4 100644 --- a/packages/service/support/outLink/schema.ts +++ b/packages/service/support/outLink/schema.ts @@ -51,6 +51,9 @@ const OutLinkSchema = new Schema({ type: Boolean, default: true }, + // showFullText: { + // type: Boolean + // }, showRawSource: { type: Boolean }, diff --git a/packages/web/common/fetch/type.d.ts b/packages/web/common/fetch/type.d.ts index 7f2ea1fbf..b297443e5 100644 --- a/packages/web/common/fetch/type.d.ts +++ b/packages/web/common/fetch/type.d.ts @@ -11,3 +11,22 @@ type PaginationResponse = { total: number; list: T[]; }; + +type LinkedPaginationProps = T & { + pageSize: number; +} & RequireOnlyOne<{ + initialId: string; + nextId: string; + prevId: string; + }> & + RequireOnlyOne<{ + initialIndex: number; + nextIndex: number; + prevIndex: number; + }>; + +type LinkedListResponse = { + list: Array; + hasMorePrev: boolean; + hasMoreNext: boolean; +}; diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index 5511c1fb1..1a91861de 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -35,10 +35,13 @@ export const iconPaths = { 'common/dingtalkFill': () => import('./icons/common/dingtalkFill.svg'), 'common/disable': () => import('./icons/common/disable.svg'), 'common/downArrowFill': () => import('./icons/common/downArrowFill.svg'), + 'common/download': () => import('./icons/common/download.svg'), + 'common/edit': () => import('./icons/common/edit.svg'), 'common/editor/resizer': () => import('./icons/common/editor/resizer.svg'), 'common/enable': () => import('./icons/common/enable.svg'), 'common/errorFill': () => import('./icons/common/errorFill.svg'), 'common/file/move': () => import('./icons/common/file/move.svg'), + 'common/fileNotFound': () => import('./icons/common/fileNotFound.svg'), 'common/folderFill': () => import('./icons/common/folderFill.svg'), 'common/folderImport': () => import('./icons/common/folderImport.svg'), 'common/fullScreenLight': () => import('./icons/common/fullScreenLight.svg'), @@ -86,7 +89,9 @@ export const iconPaths = { 'common/selectLight': () => import('./icons/common/selectLight.svg'), 'common/setting': () => import('./icons/common/setting.svg'), 'common/settingLight': () => import('./icons/common/settingLight.svg'), + 'common/solidChevronDown': () => import('./icons/common/solidChevronDown.svg'), 'common/solidChevronRight': () => import('./icons/common/solidChevronRight.svg'), + 'common/solidChevronUp': () => import('./icons/common/solidChevronUp.svg'), 'common/subtract': () => import('./icons/common/subtract.svg'), 'common/templateMarket': () => import('./icons/common/templateMarket.svg'), 'common/text/t': () => import('./icons/common/text/t.svg'), @@ -96,6 +101,7 @@ export const iconPaths = { 'common/trash': () => import('./icons/common/trash.svg'), 'common/upRightArrowLight': () => import('./icons/common/upRightArrowLight.svg'), 'common/uploadFileFill': () => import('./icons/common/uploadFileFill.svg'), + 'common/upperRight': () => import('./icons/common/upperRight.svg'), 'common/userInfo': () => import('./icons/common/userInfo.svg'), 'common/variable': () => import('./icons/common/variable.svg'), 'common/viewLight': () => import('./icons/common/viewLight.svg'), diff --git a/packages/web/components/common/Icon/icons/common/download.svg b/packages/web/components/common/Icon/icons/common/download.svg new file mode 100644 index 000000000..f70f2fbf0 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/download.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/edit.svg b/packages/web/components/common/Icon/icons/common/edit.svg new file mode 100644 index 000000000..da3bc70b8 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/edit.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/fileNotFound.svg b/packages/web/components/common/Icon/icons/common/fileNotFound.svg new file mode 100644 index 000000000..b30ce1322 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/fileNotFound.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/solidChevronDown.svg b/packages/web/components/common/Icon/icons/common/solidChevronDown.svg new file mode 100644 index 000000000..20f946546 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/solidChevronDown.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/solidChevronUp.svg b/packages/web/components/common/Icon/icons/common/solidChevronUp.svg new file mode 100644 index 000000000..2c69615b6 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/solidChevronUp.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/upperRight.svg b/packages/web/components/common/Icon/icons/common/upperRight.svg new file mode 100644 index 000000000..33bee25b1 --- /dev/null +++ b/packages/web/components/common/Icon/icons/common/upperRight.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/chat/quoteFill.svg b/packages/web/components/common/Icon/icons/core/chat/quoteFill.svg index e09f3fc29..2d5dbee3c 100644 --- a/packages/web/components/common/Icon/icons/core/chat/quoteFill.svg +++ b/packages/web/components/common/Icon/icons/core/chat/quoteFill.svg @@ -1 +1,3 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/hooks/useLinkedScroll.tsx b/packages/web/hooks/useLinkedScroll.tsx new file mode 100644 index 000000000..d612a1cad --- /dev/null +++ b/packages/web/hooks/useLinkedScroll.tsx @@ -0,0 +1,307 @@ +import { useCallback, useEffect, useRef, useState, ReactNode } from 'react'; +import { LinkedListResponse, LinkedPaginationProps } from '../common/fetch/type'; +import { Box, BoxProps } from '@chakra-ui/react'; +import { useTranslation } from 'next-i18next'; +import { useScroll, useMemoizedFn, useDebounceEffect } from 'ahooks'; +import MyBox from '../components/common/MyBox'; +import { useRequest2 } from './useRequest'; + +const threshold = 100; + +export function useLinkedScroll< + TParams extends LinkedPaginationProps & { isInitialLoad?: boolean }, + TData extends LinkedListResponse +>( + api: (data: TParams) => Promise, + { + refreshDeps = [], + pageSize = 15, + params = {}, + initialId, + initialIndex, + canLoadData = false + }: { + refreshDeps?: any[]; + pageSize?: number; + params?: Record; + initialId?: string; + initialIndex?: number; + canLoadData?: boolean; + } +) { + const { t } = useTranslation(); + const [dataList, setDataList] = useState([]); + const [hasMorePrev, setHasMorePrev] = useState(true); + const [hasMoreNext, setHasMoreNext] = useState(true); + const [initialLoadDone, setInitialLoadDone] = useState(false); + const hasScrolledToInitial = useRef(false); + + const anchorRef = useRef({ + top: null as { _id: string; index: number } | null, + bottom: null as { _id: string; index: number } | null + }); + + const containerRef = useRef(null); + const itemRefs = useRef<(HTMLElement | null)[]>([]); + + const { runAsync: callApi, loading: isLoading } = useRequest2( + async (apiParams: TParams) => await api(apiParams), + { + onError: (error) => { + return Promise.reject(error); + } + } + ); + + const loadData = useCallback( + async ({ + id, + index, + isInitialLoad = false + }: { + id: string; + index: number; + isInitialLoad?: boolean; + }) => { + if (isLoading) return null; + + const response = await callApi({ + initialId: id, + initialIndex: index, + pageSize, + isInitialLoad, + ...params + } as TParams); + + if (!response) return null; + + setHasMorePrev(response.hasMorePrev); + setHasMoreNext(response.hasMoreNext); + setDataList(response.list); + + if (response.list.length > 0) { + anchorRef.current.top = response.list[0]; + anchorRef.current.bottom = response.list[response.list.length - 1]; + } + + setInitialLoadDone(true); + + const scrollIndex = response.list.findIndex((item) => item._id === id); + + if (scrollIndex !== -1 && itemRefs.current?.[scrollIndex]) { + setTimeout(() => { + scrollToItem(scrollIndex); + }, 100); + } + + return response; + }, + [callApi, params, dataList, hasMorePrev, hasMoreNext, isLoading] + ); + + const loadPrevData = useCallback( + async (scrollRef = containerRef) => { + if (!anchorRef.current.top || !hasMorePrev || isLoading) return; + + const prevScrollTop = scrollRef?.current?.scrollTop || 0; + const prevScrollHeight = scrollRef?.current?.scrollHeight || 0; + + const response = await callApi({ + prevId: anchorRef.current.top._id, + prevIndex: anchorRef.current.top.index, + pageSize, + ...params + } as TParams); + + if (!response) return; + + setHasMorePrev(response.hasMorePrev); + + if (response.list.length > 0) { + setDataList((prev) => [...response.list, ...prev]); + anchorRef.current.top = response.list[0]; + + setTimeout(() => { + if (scrollRef?.current) { + const newHeight = scrollRef.current.scrollHeight; + const heightDiff = newHeight - prevScrollHeight; + scrollRef.current.scrollTop = prevScrollTop + heightDiff; + } + }, 0); + } + + return response; + }, + [callApi, hasMorePrev, isLoading, params, pageSize] + ); + + const loadNextData = useCallback( + async (scrollRef = containerRef) => { + if (!anchorRef.current.bottom || !hasMoreNext || isLoading) return; + + const prevScrollTop = scrollRef?.current?.scrollTop || 0; + + const response = await callApi({ + nextId: anchorRef.current.bottom._id, + nextIndex: anchorRef.current.bottom.index, + pageSize, + ...params + } as TParams); + + if (!response) return; + + setHasMoreNext(response.hasMoreNext); + + if (response.list.length > 0) { + setDataList((prev) => [...prev, ...response.list]); + anchorRef.current.bottom = response.list[response.list.length - 1]; + + setTimeout(() => { + if (scrollRef?.current) { + scrollRef.current.scrollTop = prevScrollTop; + } + }, 0); + } + + return response; + }, + [callApi, hasMoreNext, isLoading, params, pageSize] + ); + + const scrollToItem = useCallback( + (itemIndex: number) => { + if (itemIndex >= 0 && itemIndex < dataList.length && itemRefs.current?.[itemIndex]) { + try { + const element = itemRefs.current[itemIndex]; + if (!element) { + return false; + } + + setTimeout(() => { + if (element && containerRef.current) { + const elementRect = element.getBoundingClientRect(); + const containerRect = containerRef.current.getBoundingClientRect(); + + const relativeTop = elementRect.top - containerRect.top; + + const scrollTop = + containerRef.current.scrollTop + + relativeTop - + containerRect.height / 2 + + elementRect.height / 2; + + containerRef.current.scrollTo({ + top: scrollTop, + behavior: 'smooth' + }); + } + }, 50); + + return true; + } catch (error) { + console.error('Error scrolling to item:', error); + return false; + } + } + return false; + }, + [dataList.length] + ); + + // 初始加载 + useEffect(() => { + if (canLoadData) { + // 重置初始滚动状态 + hasScrolledToInitial.current = false; + + loadData({ + id: initialId || '', + index: initialIndex || 0, + isInitialLoad: true + }); + } + }, [canLoadData, ...refreshDeps]); + + // 监听初始加载完成，执行初始滚动 + useEffect(() => { + if (initialLoadDone && dataList.length > 0 && !hasScrolledToInitial.current) { + hasScrolledToInitial.current = true; + const foundIndex = dataList.findIndex((item) => item._id === initialId); + + if (foundIndex >= 0) { + setTimeout(() => { + scrollToItem(foundIndex); + }, 200); + } + } + }, [initialLoadDone, ...refreshDeps]); + + const ScrollData = useMemoizedFn( + ({ + children, + ScrollContainerRef, + isLoading: externalLoading, + ...props + }: { + isLoading?: boolean; + children: ReactNode; + ScrollContainerRef?: React.RefObject; + } & BoxProps) => { + const ref = ScrollContainerRef || containerRef; + const scroll = useScroll(ref); + + useDebounceEffect( + () => { + if (!ref?.current || isLoading || !initialLoadDone) return; + + const { scrollTop, scrollHeight, clientHeight } = ref.current; + + // 滚动到底部附近，加载更多下方数据 + if (scrollTop + clientHeight >= scrollHeight - threshold && hasMoreNext) { + loadNextData(ref); + } + + // 滚动到顶部附近，加载更多上方数据 + if (scrollTop <= threshold && hasMorePrev) { + loadPrevData(ref); + } + }, + [scroll], + { wait: 200 } + ); + + return ( + + {hasMorePrev && isLoading && initialLoadDone && ( + + {t('common:common.is_requesting')} + + )} + {children} + {hasMoreNext && isLoading && initialLoadDone && ( + + {t('common:common.is_requesting')} + + )} + + ); + } + ); + + return { + dataList, + setDataList, + isLoading, + loadData, + initialLoadDone, + ScrollData, + itemRefs, + scrollToItem + }; +} diff --git a/packages/web/i18n/en/chat.json b/packages/web/i18n/en/chat.json index e790daf2b..037b3d08d 100644 --- a/packages/web/i18n/en/chat.json +++ b/packages/web/i18n/en/chat.json @@ -3,6 +3,8 @@ "Delete_all": "Clear All Lexicon", "LLM_model_response_empty": "The model flow response is empty, please check whether the model flow output is normal.", "ai_reasoning": "Thinking process", + "chat.quote.No Data": "The file cannot be found", + "chat.quote.deleted": "This data has been deleted ~", "chat_history": "Conversation History", "chat_input_guide_lexicon_is_empty": "Lexicon not configured yet", "chat_test_app": "Debug-{{name}}", diff --git a/packages/web/i18n/en/common.json b/packages/web/i18n/en/common.json index 0e4cc97c7..e81162a04 100644 --- a/packages/web/i18n/en/common.json +++ b/packages/web/i18n/en/common.json @@ -1,5 +1,6 @@ { "App": "Application", + "Download": "Download", "Export": "Export", "FAQ.ai_point_a": "Each time you use the AI model, a certain amount of AI points will be deducted. For detailed calculation standards, please refer to the 'AI Points Calculation Standards' above.\nToken calculation uses the same formula as GPT-3.5, where 1 Token ≈ 0.7 Chinese characters ≈ 0.9 English words. Consecutive characters may be considered as 1 Token.", "FAQ.ai_point_expire_a": "Yes, they will expire. After the current package expires, the AI points will be reset to the new package's AI points. Annual package AI points are valid for one year, not monthly.", @@ -450,6 +451,8 @@ "core.chat.module_unexist": "Running failed: Application missing components", "core.chat.quote.Quote Tip": "Only the actual quoted content is displayed here. If the data is updated, it will not be updated in real-time here.", "core.chat.quote.Read Quote": "View Quote", + "core.chat.quote.afterUpdate": "After update", + "core.chat.quote.beforeUpdate": "Before update", "core.chat.response.Complete Response": "Complete Response", "core.chat.response.Extension model": "Question Optimization Model", "core.chat.response.Read complete response": "View Details", @@ -495,9 +498,11 @@ "core.dataset.Dataset": "Dataset", "core.dataset.Dataset ID": "Dataset ID", "core.dataset.Delete Confirm": "Confirm to Delete This Dataset? Data Cannot Be Recovered After Deletion, Please Confirm!", + "core.dataset.Download the parsed content": "Download the parsed content", "core.dataset.Empty Dataset": "Empty Dataset", "core.dataset.Empty Dataset Tips": "No Dataset Yet, Create One Now!", "core.dataset.Folder placeholder": "This is a Directory", + "core.dataset.Get the raw data": "Get the raw data", "core.dataset.Go Dataset": "Go to Dataset", "core.dataset.Intro Placeholder": "This Dataset Has No Introduction Yet", "core.dataset.Manual collection": "Manual Dataset", @@ -540,6 +545,7 @@ "core.dataset.data.Empty Tip": "This collection has no data yet", "core.dataset.data.Search data placeholder": "Search Related Data", "core.dataset.data.Too Long": "Total Length Exceeded", + "core.dataset.data.Updated": "Updated", "core.dataset.data.group": "Group", "core.dataset.data.unit": "Items", "core.dataset.embedding model tip": "The index model can convert natural language into vectors for semantic search.\nNote that different index models cannot be used together. Once an index model is selected, it cannot be changed.", @@ -1027,6 +1033,8 @@ "support.outlink.Max usage points": "Points Limit", "support.outlink.Max usage points tip": "The maximum number of points allowed for this link. It cannot be used after exceeding the limit. -1 means unlimited.", "support.outlink.Usage points": "Points Consumption", + "support.outlink.share.Chat_quote_reader": "Full text reader", + "support.outlink.share.Full_text tips": "Allows reading of the complete dataset from which the referenced fragment is derived", "support.outlink.share.Response Quote": "Return Quote", "support.outlink.share.Response Quote tips": "Return quoted content in the share link, but do not allow users to download the original document", "support.outlink.share.running_node": "Running node", diff --git a/packages/web/i18n/zh-CN/chat.json b/packages/web/i18n/zh-CN/chat.json index 098f1e01b..a8cefa2db 100644 --- a/packages/web/i18n/zh-CN/chat.json +++ b/packages/web/i18n/zh-CN/chat.json @@ -3,6 +3,8 @@ "Delete_all": "清空词库", "LLM_model_response_empty": "模型流响应为空，请检查模型流输出是否正常", "ai_reasoning": "思考过程", + "chat.quote.No Data": "找不到该文件", + "chat.quote.deleted": "该数据已被删除～", "chat_history": "聊天记录", "chat_input_guide_lexicon_is_empty": "还没有配置词库", "chat_test_app": "调试-{{name}}", diff --git a/packages/web/i18n/zh-CN/common.json b/packages/web/i18n/zh-CN/common.json index bc119e67c..8eaed9e0a 100644 --- a/packages/web/i18n/zh-CN/common.json +++ b/packages/web/i18n/zh-CN/common.json @@ -1,5 +1,6 @@ { "App": "应用", + "Download": "下载", "Export": "导出", "FAQ.ai_point_a": "每次调用AI模型时，都会消耗一定的AI积分。具体的计算标准可参考上方的“AI 积分计算标准”。\nToken计算采用GPT3.5相同公式，1Token≈0.7中文字符≈0.9英文单词，连续出现的字符可能被认为是1个Tokens。", "FAQ.ai_point_expire_a": "会过期。当前套餐过期后，AI积分将会清空，并更新为新套餐的AI积分。年度套餐的AI积分时长为1年，而不是每个月。", @@ -453,6 +454,8 @@ "core.chat.module_unexist": "运行失败：应用缺失组件", "core.chat.quote.Quote Tip": "此处仅显示实际引用内容，若数据有更新，此处不会实时更新", "core.chat.quote.Read Quote": "查看引用", + "core.chat.quote.afterUpdate": "更新后", + "core.chat.quote.beforeUpdate": "更新前", "core.chat.response.Complete Response": "完整响应", "core.chat.response.Extension model": "问题优化模型", "core.chat.response.Read complete response": "查看详情", @@ -498,9 +501,11 @@ "core.dataset.Dataset": "知识库", "core.dataset.Dataset ID": "知识库 ID", "core.dataset.Delete Confirm": "确认删除该知识库？删除后数据无法恢复，请确认！", + "core.dataset.Download the parsed content": "下载解析内容", "core.dataset.Empty Dataset": "空数据集", "core.dataset.Empty Dataset Tips": "还没有知识库，快去创建一个吧！", "core.dataset.Folder placeholder": "这是一个目录", + "core.dataset.Get the raw data": "获取源数据", "core.dataset.Go Dataset": "前往知识库", "core.dataset.Intro Placeholder": "这个知识库还没有介绍~", "core.dataset.Manual collection": "手动数据集", @@ -543,6 +548,7 @@ "core.dataset.data.Empty Tip": "这个集合还没有数据~", "core.dataset.data.Search data placeholder": "搜索相关数据", "core.dataset.data.Too Long": "总长度超长了", + "core.dataset.data.Updated": "已更新", "core.dataset.data.group": "组", "core.dataset.data.unit": "条", "core.dataset.embedding model tip": "索引模型可以将自然语言转成向量，用于进行语义检索。\n注意，不同索引模型无法一起使用，选择完索引模型后将无法修改。", @@ -1031,6 +1037,8 @@ "support.outlink.Max usage points": "积分上限", "support.outlink.Max usage points tip": "该链接最多允许使用多少积分，超出后将无法使用。-1 代表无限制。", "support.outlink.Usage points": "积分消耗", + "support.outlink.share.Chat_quote_reader": "全文阅读器", + "support.outlink.share.Full_text tips": "允许阅读该引用片段来源的完整数据集", "support.outlink.share.Response Quote": "引用内容", "support.outlink.share.Response Quote tips": "查看知识库搜索的引用内容，不可查看完整引用文档或跳转引用网站", "support.outlink.share.running_node": "运行节点", diff --git a/packages/web/i18n/zh-Hant/chat.json b/packages/web/i18n/zh-Hant/chat.json index 24f28ef78..b7494de5f 100644 --- a/packages/web/i18n/zh-Hant/chat.json +++ b/packages/web/i18n/zh-Hant/chat.json @@ -3,6 +3,8 @@ "Delete_all": "清除所有詞彙", "LLM_model_response_empty": "模型流程回應為空，請檢查模型流程輸出是否正常", "ai_reasoning": "思考過程", + "chat.quote.No Data": "找不到該文件", + "chat.quote.deleted": "該數據已被刪除～", "chat_history": "對話紀錄", "chat_input_guide_lexicon_is_empty": "尚未設定詞彙庫", "chat_test_app": "調試-{{name}}", diff --git a/packages/web/i18n/zh-Hant/common.json b/packages/web/i18n/zh-Hant/common.json index 4fb5a389c..c6a3771c2 100644 --- a/packages/web/i18n/zh-Hant/common.json +++ b/packages/web/i18n/zh-Hant/common.json @@ -1,5 +1,6 @@ { "App": "應用程式", + "Download": "下載", "Export": "匯出", "FAQ.ai_point_a": "每次呼叫 AI 模型時，都會消耗一定數量的 AI 點數。詳細的計算標準請參考上方的「AI 點數計算標準」。\nToken 計算採用與 GPT3.5 相同的公式，1 Token ≈ 0.7 個中文字 ≈ 0.9 個英文單字，連續出現的字元可能會被視為 1 個 Token。", "FAQ.ai_point_expire_a": "會過期。目前方案過期後，AI 點數將會清空並更新為新方案的 AI 點數。年度方案的 AI 點數有效期為一年，而不是每個月重置。", @@ -449,6 +450,8 @@ "core.chat.module_unexist": "運行失敗：應用缺失組件", "core.chat.quote.Quote Tip": "此處僅顯示實際引用內容，若資料有更新，此處不會即時更新", "core.chat.quote.Read Quote": "檢視引用", + "core.chat.quote.afterUpdate": "更新後", + "core.chat.quote.beforeUpdate": "更新前", "core.chat.response.Complete Response": "完整回應", "core.chat.response.Extension model": "問題最佳化模型", "core.chat.response.Read complete response": "檢視詳細資料", @@ -494,9 +497,11 @@ "core.dataset.Dataset": "知識庫", "core.dataset.Dataset ID": "知識庫 ID", "core.dataset.Delete Confirm": "確認刪除此知識庫？刪除後資料無法復原，請確認！", + "core.dataset.Download the parsed content": "下載解析內容", "core.dataset.Empty Dataset": "空資料集", "core.dataset.Empty Dataset Tips": "還沒有知識庫，快來建立一個吧！", "core.dataset.Folder placeholder": "這是一個目錄", + "core.dataset.Get the raw data": "獲取源數據", "core.dataset.Go Dataset": "前往知識庫", "core.dataset.Intro Placeholder": "這個知識庫還沒有介紹", "core.dataset.Manual collection": "手動資料集", @@ -539,6 +544,7 @@ "core.dataset.data.Empty Tip": "此集合還沒有資料", "core.dataset.data.Search data placeholder": "搜尋相關資料", "core.dataset.data.Too Long": "總長度超出上限", + "core.dataset.data.Updated": "已更新", "core.dataset.data.group": "組", "core.dataset.data.unit": "筆", "core.dataset.embedding model tip": "索引模型可以將自然語言轉換成向量，用於進行語意搜尋。\n注意，不同索引模型無法一起使用。選擇索引模型後就無法修改。", @@ -1026,6 +1032,8 @@ "support.outlink.Max usage points": "點數上限", "support.outlink.Max usage points tip": "此連結最多允許使用多少點數，超出後將無法使用。-1 代表無限制。", "support.outlink.Usage points": "點數消耗", + "support.outlink.share.Chat_quote_reader": "全文閱讀器", + "support.outlink.share.Full_text tips": "允許閱讀該引用片段來源的完整數據集", "support.outlink.share.Response Quote": "回傳引用", "support.outlink.share.Response Quote tips": "在分享連結中回傳引用內容，但不允許使用者下載原始文件", "support.outlink.share.running_node": "執行節點", diff --git a/projects/app/src/components/SideBar/index.tsx b/projects/app/src/components/SideBar/index.tsx index 9a286f2f4..27bd64861 100644 --- a/projects/app/src/components/SideBar/index.tsx +++ b/projects/app/src/components/SideBar/index.tsx @@ -1,22 +1,41 @@ -import React, { useState } from 'react'; +import React, { useState, useEffect, useRef } from 'react'; import { Box, Flex } from '@chakra-ui/react'; import type { BoxProps } from '@chakra-ui/react'; import MyIcon from '@fastgpt/web/components/common/Icon'; -interface Props extends BoxProps {} +interface Props extends BoxProps { + externalTrigger?: Boolean; +} const SideBar = (e?: Props) => { const { w = ['100%', '0 0 250px', '0 0 270px', '0 0 290px', '0 0 310px'], children, + externalTrigger, ...props } = e || {}; - const [foldSideBar, setFoldSideBar] = useState(false); + const [isFolded, setIsFolded] = useState(false); + + const prevExternalTriggerRef = useRef(undefined); + + useEffect(() => { + if (externalTrigger && !prevExternalTriggerRef.current && !isFolded) { + setIsFolded(true); + } + + prevExternalTriggerRef.current = externalTrigger; + }, [externalTrigger, isFolded]); + + const handleToggle = () => { + const newFolded = !isFolded; + setIsFolded(newFolded); + }; + return ( { bg={'rgba(0,0,0,0.5)'} cursor={'pointer'} transition={'0.2s'} - {...(foldSideBar + {...(isFolded ? { opacity: 0.6 } @@ -48,16 +67,16 @@ const SideBar = (e?: Props) => { visibility: 'hidden', opacity: 0 })} - onClick={() => setFoldSideBar(!foldSideBar)} + onClick={handleToggle} > - + {children} diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/QuoteList.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/QuoteList.tsx new file mode 100644 index 000000000..672c97db9 --- /dev/null +++ b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/QuoteList.tsx @@ -0,0 +1,96 @@ +import React, { useMemo } from 'react'; +import { Box, useTheme } from '@chakra-ui/react'; + +import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; +import QuoteItem, { formatScore } from '@/components/core/dataset/QuoteItem'; +import { useContextSelector } from 'use-context-selector'; +import { ChatBoxContext } from '../Provider'; +import { ChatItemContext } from '@/web/core/chat/context/chatItemContext'; +import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; +import { useChatStore } from '@/web/core/chat/context/useChatStore'; +import { getQuoteDataList } from '@/web/core/chat/api'; + +const QuoteList = React.memo(function QuoteList({ + chatItemId, + rawSearch = [], + chatTime +}: { + chatItemId?: string; + rawSearch: SearchDataResponseItemType[]; + chatTime: Date; +}) { + const theme = useTheme(); + const { chatId, appId, outLinkAuthData } = useChatStore(); + + const RawSourceBoxProps = useContextSelector(ChatBoxContext, (v) => ({ + chatItemId, + appId: v.appId, + chatId: v.chatId, + ...(v.outLinkAuthData || {}) + })); + const showRawSource = useContextSelector(ChatItemContext, (v) => v.isShowReadRawSource); + const showRouteToDatasetDetail = useContextSelector( + ChatItemContext, + (v) => v.showRouteToDatasetDetail + ); + + const { data } = useRequest2( + async () => + await getQuoteDataList({ + datasetDataIdList: rawSearch.map((item) => item.id), + chatTime, + collectionIdList: [...new Set(rawSearch.map((item) => item.collectionId))], + chatItemId: chatItemId || '', + appId, + chatId, + ...outLinkAuthData + }), + { + manual: false + } + ); + + const formatedDataList = useMemo(() => { + return rawSearch + .map((item) => { + const currentFilterItem = data?.quoteList.find((res) => res._id === item.id); + + return { + ...item, + q: currentFilterItem?.q || '', + a: currentFilterItem?.a || '' + }; + }) + .sort((a, b) => { + const aScore = formatScore(a.score); + const bScore = formatScore(b.score); + return (bScore.primaryScore?.value || 0) - (aScore.primaryScore?.value || 0); + }); + }, [data?.quoteList, rawSearch]); + + return ( + <> + {formatedDataList.map((item, i) => ( + + + + ))} + + ); +}); + +export default QuoteList; diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/QuoteModal.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/QuoteModal.tsx deleted file mode 100644 index a8457d0b0..000000000 --- a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/QuoteModal.tsx +++ /dev/null @@ -1,129 +0,0 @@ -import React, { useMemo } from 'react'; -import { ModalBody, Box, useTheme } from '@chakra-ui/react'; - -import MyModal from '@fastgpt/web/components/common/MyModal'; -import { useTranslation } from 'next-i18next'; -import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; -import QuoteItem from '@/components/core/dataset/QuoteItem'; -import RawSourceBox from '@/components/core/dataset/RawSourceBox'; -import { getWebReqUrl } from '@fastgpt/web/common/system/utils'; -import { useContextSelector } from 'use-context-selector'; -import { ChatBoxContext } from '../Provider'; -import { ChatItemContext } from '@/web/core/chat/context/chatItemContext'; - -const QuoteModal = ({ - rawSearch = [], - onClose, - chatItemId, - metadata -}: { - rawSearch: SearchDataResponseItemType[]; - onClose: () => void; - chatItemId: string; - metadata?: { - collectionId: string; - sourceId?: string; - sourceName: string; - }; -}) => { - const { t } = useTranslation(); - const filterResults = useMemo( - () => - metadata - ? rawSearch.filter( - (item) => - item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId - ) - : rawSearch, - [metadata, rawSearch] - ); - - const RawSourceBoxProps = useContextSelector(ChatBoxContext, (v) => ({ - appId: v.appId, - chatId: v.chatId, - chatItemId, - ...(v.outLinkAuthData || {}) - })); - const showRawSource = useContextSelector(ChatItemContext, (v) => v.isShowReadRawSource); - const showRouteToDatasetDetail = useContextSelector( - ChatItemContext, - (v) => v.showRouteToDatasetDetail - ); - - return ( - <> - - {metadata ? ( - - ) : ( - <>{t('common:core.chat.Quote Amount', { amount: rawSearch.length })} - )} - - {t('common:core.chat.quote.Quote Tip')} - - - } - > - - - - - - ); -}; - -export default QuoteModal; - -export const QuoteList = React.memo(function QuoteList({ - chatItemId, - rawSearch = [] -}: { - chatItemId?: string; - rawSearch: SearchDataResponseItemType[]; -}) { - const theme = useTheme(); - - const RawSourceBoxProps = useContextSelector(ChatBoxContext, (v) => ({ - chatItemId, - appId: v.appId, - chatId: v.chatId, - ...(v.outLinkAuthData || {}) - })); - const showRawSource = useContextSelector(ChatItemContext, (v) => v.isShowReadRawSource); - const showRouteToDatasetDetail = useContextSelector( - ChatItemContext, - (v) => v.showRouteToDatasetDetail - ); - - return ( - <> - {rawSearch.map((item, i) => ( - - - - ))} - - ); -}); diff --git a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ResponseTags.tsx b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ResponseTags.tsx index 7be7111e5..9655673e0 100644 --- a/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ResponseTags.tsx +++ b/projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ResponseTags.tsx @@ -14,8 +14,8 @@ import { addStatisticalDataToHistoryItem } from '@/global/core/chat/utils'; import { useSize } from 'ahooks'; import { useContextSelector } from 'use-context-selector'; import { ChatBoxContext } from '../Provider'; +import { ChatItemContext } from '@/web/core/chat/context/chatItemContext'; -const QuoteModal = dynamic(() => import('./QuoteModal')); const ContextModal = dynamic(() => import('./ContextModal')); const WholeResponseModal = dynamic(() => import('../../../components/WholeResponseModal')); @@ -30,6 +30,7 @@ const ResponseTags = ({ const { t } = useTranslation(); const quoteListRef = React.useRef(null); const dataId = historyItem.dataId; + const chatTime = historyItem.time || new Date(); const { totalQuoteList: quoteList = [], @@ -38,17 +39,12 @@ const ResponseTags = ({ historyPreviewLength = 0 } = useMemo(() => addStatisticalDataToHistoryItem(historyItem), [historyItem]); - const [quoteModalData, setQuoteModalData] = useState<{ - rawSearch: SearchDataResponseItemType[]; - metadata?: { - collectionId: string; - sourceId?: string; - sourceName: string; - }; - }>(); const [quoteFolded, setQuoteFolded] = useState(true); const chatType = useContextSelector(ChatBoxContext, (v) => v.chatType); + + const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData); + const notSharePage = useMemo(() => chatType !== 'share', [chatType]); const { @@ -81,7 +77,8 @@ const ResponseTags = ({ sourceName: item.sourceName, sourceId: item.sourceId, icon: getSourceNameIcon({ sourceId: item.sourceId, sourceName: item.sourceName }), - collectionId: item.collectionId + collectionId: item.collectionId, + datasetId: item.datasetId })); }, [quoteList]); @@ -99,7 +96,11 @@ const ResponseTags = ({ <> - + {quoteFolded && quoteIsOverflow && ( - {sourceList.map((item) => { + {sourceList.map((item, index) => { return ( { e.stopPropagation(); - setQuoteModalData({ + + setQuoteData({ + chatTime, rawSearch: quoteList, metadata: { collectionId: item.collectionId, - sourceId: item.sourceId, - sourceName: item.sourceName + collectionIdList: [ + ...new Set(quoteList.map((item) => item.collectionId)) + ], + sourceId: item.sourceId || '', + sourceName: item.sourceName, + datasetId: item.datasetId, + chatItemId: historyItem.dataId } }); }} + height={6} > - - - {item.sourceName} - + + {index + 1} + + + + + {item.sourceName} + + ); @@ -196,7 +221,22 @@ const ResponseTags = ({ colorSchema="blue" type="borderSolid" cursor={'pointer'} - onClick={() => setQuoteModalData({ rawSearch: quoteList })} + onClick={(e) => { + e.stopPropagation(); + + setQuoteData({ + chatTime, + rawSearch: quoteList, + metadata: { + collectionId: '', + collectionIdList: [...new Set(quoteList.map((item) => item.collectionId))], + chatItemId: historyItem.dataId, + sourceId: '', + sourceName: '', + datasetId: '' + } + }); + }} > {t('chat:citations', { num: quoteList.length })} @@ -246,15 +286,10 @@ const ResponseTags = ({ )} - {!!quoteModalData && ( - setQuoteModalData(undefined)} - /> - )} {isOpenContextModal && } - {isOpenWholeModal && } + {isOpenWholeModal && ( + + )} ); }; diff --git a/projects/app/src/components/core/chat/ChatContainer/PluginRunBox/components/RenderResponseDetail.tsx b/projects/app/src/components/core/chat/ChatContainer/PluginRunBox/components/RenderResponseDetail.tsx index a7725935e..5cd73279d 100644 --- a/projects/app/src/components/core/chat/ChatContainer/PluginRunBox/components/RenderResponseDetail.tsx +++ b/projects/app/src/components/core/chat/ChatContainer/PluginRunBox/components/RenderResponseDetail.tsx @@ -12,12 +12,18 @@ const RenderResponseDetail = () => { const isChatting = useContextSelector(PluginRunContext, (v) => v.isChatting); const responseData = chatRecords?.[1]?.responseData || []; + const chatTime = new Date(); return isChatting ? ( <>{t('chat:in_progress')} ) : ( - + ); }; diff --git a/projects/app/src/components/core/chat/Divider/index.tsx b/projects/app/src/components/core/chat/Divider/index.tsx index 39d4b7718..50b57863a 100644 --- a/projects/app/src/components/core/chat/Divider/index.tsx +++ b/projects/app/src/components/core/chat/Divider/index.tsx @@ -3,11 +3,19 @@ import React from 'react'; import MyIcon from '@fastgpt/web/components/common/Icon'; import type { IconNameType } from '@fastgpt/web/components/common/Icon/type.d'; -const ChatBoxDivider = ({ icon, text }: { icon: IconNameType; text: string }) => { +const ChatBoxDivider = ({ + icon, + text, + iconColor +}: { + icon: IconNameType; + text: string; + iconColor?: string; +}) => { return ( - + {text} diff --git a/projects/app/src/components/core/chat/components/WholeResponseModal.tsx b/projects/app/src/components/core/chat/components/WholeResponseModal.tsx index b01870bb0..d51a7a1e3 100644 --- a/projects/app/src/components/core/chat/components/WholeResponseModal.tsx +++ b/projects/app/src/components/core/chat/components/WholeResponseModal.tsx @@ -5,7 +5,7 @@ import { useTranslation } from 'next-i18next'; import { moduleTemplatesFlat } from '@fastgpt/global/core/workflow/template/constants'; import MyModal from '@fastgpt/web/components/common/MyModal'; import Markdown from '@/components/Markdown'; -import { QuoteList } from '../ChatContainer/ChatBox/components/QuoteModal'; +import QuoteList from '../ChatContainer/ChatBox/components/QuoteList'; import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constants'; import { formatNumber } from '@fastgpt/global/common/math/tools'; import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; @@ -32,11 +32,13 @@ type sideTabItemType = { export const WholeResponseContent = ({ activeModule, hideTabs, - dataId + dataId, + chatTime }: { activeModule: ChatHistoryItemResType; hideTabs?: boolean; dataId?: string; + chatTime?: Date; }) => { const { t } = useTranslation(); @@ -263,7 +265,13 @@ export const WholeResponseContent = ({ {activeModule.quoteList && activeModule.quoteList.length > 0 && ( } + rawDom={ + + } /> )} @@ -562,11 +570,13 @@ const SideTabItem = ({ export const ResponseBox = React.memo(function ResponseBox({ response, dataId, + chatTime, hideTabs = false, useMobile = false }: { response: ChatHistoryItemResType[]; dataId?: string; + chatTime: Date; hideTabs?: boolean; useMobile?: boolean; }) { @@ -689,7 +699,12 @@ export const ResponseBox = React.memo(function ResponseBox({ - + ) : ( @@ -753,6 +768,7 @@ export const ResponseBox = React.memo(function ResponseBox({ dataId={dataId} activeModule={activeModule} hideTabs={hideTabs} + chatTime={chatTime} /> @@ -763,7 +779,15 @@ export const ResponseBox = React.memo(function ResponseBox({ ); }); -const WholeResponseModal = ({ onClose, dataId }: { onClose: () => void; dataId: string }) => { +const WholeResponseModal = ({ + onClose, + dataId, + chatTime +}: { + onClose: () => void; + dataId: string; + chatTime: Date; +}) => { const { t } = useTranslation(); const { getHistoryResponseData } = useContextSelector(ChatBoxContext, (v) => v); @@ -792,7 +816,7 @@ const WholeResponseModal = ({ onClose, dataId }: { onClose: () => void; dataId: } > {!!response?.length ? ( - + ) : ( )} diff --git a/projects/app/src/components/core/dataset/QuoteItem.tsx b/projects/app/src/components/core/dataset/QuoteItem.tsx index a7c2f9261..afd158d4b 100644 --- a/projects/app/src/components/core/dataset/QuoteItem.tsx +++ b/projects/app/src/components/core/dataset/QuoteItem.tsx @@ -14,8 +14,8 @@ import Markdown from '@/components/Markdown'; const InputDataModal = dynamic(() => import('@/pageComponents/dataset/detail/InputDataModal')); -type ScoreItemType = SearchDataResponseItemType['score'][0]; -const scoreTheme: Record< +export type ScoreItemType = SearchDataResponseItemType['score'][0]; +export const scoreTheme: Record< string, { color: string; @@ -44,6 +44,47 @@ const scoreTheme: Record< } }; +export const formatScore = (score: ScoreItemType[]) => { + if (!Array.isArray(score)) { + return { + primaryScore: undefined, + secondaryScore: [] + }; + } + + // rrf -> rerank -> embedding -> fullText 优先级 + let rrfScore: ScoreItemType | undefined = undefined; + let reRankScore: ScoreItemType | undefined = undefined; + let embeddingScore: ScoreItemType | undefined = undefined; + let fullTextScore: ScoreItemType | undefined = undefined; + + score.forEach((item) => { + if (item.type === SearchScoreTypeEnum.rrf) { + rrfScore = item; + } else if (item.type === SearchScoreTypeEnum.reRank) { + reRankScore = item; + } else if (item.type === SearchScoreTypeEnum.embedding) { + embeddingScore = item; + } else if (item.type === SearchScoreTypeEnum.fullText) { + fullTextScore = item; + } + }); + + const primaryScore = (rrfScore || + reRankScore || + embeddingScore || + fullTextScore) as unknown as ScoreItemType; + const secondaryScore = [rrfScore, reRankScore, embeddingScore, fullTextScore].filter( + // @ts-ignore + (item) => item && primaryScore && item.type !== primaryScore.type + ) as unknown as ScoreItemType[]; + + return { + primaryScore, + secondaryScore + }; +}; + const QuoteItem = ({ quoteItem, canViewSource, @@ -58,44 +99,7 @@ const QuoteItem = ({ const [editInputData, setEditInputData] = useState<{ dataId: string; collectionId: string }>(); const score = useMemo(() => { - if (!Array.isArray(quoteItem.score)) { - return { - primaryScore: undefined, - secondaryScore: [] - }; - } - - // rrf -> rerank -> embedding -> fullText 优先级 - let rrfScore: ScoreItemType | undefined = undefined; - let reRankScore: ScoreItemType | undefined = undefined; - let embeddingScore: ScoreItemType | undefined = undefined; - let fullTextScore: ScoreItemType | undefined = undefined; - - quoteItem.score.forEach((item) => { - if (item.type === SearchScoreTypeEnum.rrf) { - rrfScore = item; - } else if (item.type === SearchScoreTypeEnum.reRank) { - reRankScore = item; - } else if (item.type === SearchScoreTypeEnum.embedding) { - embeddingScore = item; - } else if (item.type === SearchScoreTypeEnum.fullText) { - fullTextScore = item; - } - }); - - const primaryScore = (rrfScore || - reRankScore || - embeddingScore || - fullTextScore) as unknown as ScoreItemType; - const secondaryScore = [rrfScore, reRankScore, embeddingScore, fullTextScore].filter( - // @ts-ignore - (item) => item && primaryScore && item.type !== primaryScore.type - ) as unknown as ScoreItemType[]; - - return { - primaryScore, - secondaryScore - }; + return formatScore(quoteItem.score); }, [quoteItem.score]); return ( diff --git a/projects/app/src/global/core/dataset/type.d.ts b/projects/app/src/global/core/dataset/type.d.ts index 945270b01..3eca9728b 100644 --- a/projects/app/src/global/core/dataset/type.d.ts +++ b/projects/app/src/global/core/dataset/type.d.ts @@ -39,5 +39,6 @@ export type DatasetDataListItemType = { q: string; // embedding content a: string; // bonus content chunkIndex?: number; + updated?: boolean; // indexes: DatasetDataSchemaType['indexes']; }; diff --git a/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx b/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx index 4a20b9d87..32cdebdf6 100644 --- a/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx +++ b/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx @@ -19,6 +19,7 @@ import ChatRecordContextProvider, { } from '@/web/core/chat/context/chatRecordContext'; import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; import { useContextSelector } from 'use-context-selector'; +import ChatQuoteList from '@/pageComponents/chat/ChatQuoteList'; const PluginRunBox = dynamic(() => import('@/components/core/chat/ChatContainer/PluginRunBox')); const ChatBox = dynamic(() => import('@/components/core/chat/ChatContainer/ChatBox')); @@ -37,6 +38,8 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => { const setChatBoxData = useContextSelector(ChatItemContext, (v) => v.setChatBoxData); const pluginRunTab = useContextSelector(ChatItemContext, (v) => v.pluginRunTab); const setPluginRunTab = useContextSelector(ChatItemContext, (v) => v.setPluginRunTab); + const quoteData = useContextSelector(ChatItemContext, (v) => v.quoteData); + const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData); const chatRecords = useContextSelector(ChatRecordContext, (v) => v.chatRecords); const totalRecordsCount = useContextSelector(ChatRecordContext, (v) => v.totalRecordsCount); @@ -76,7 +79,7 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => { zIndex={3} position={['fixed', 'absolute']} top={[0, '2%']} - right={0} + right={quoteData ? 600 : 0} h={['100%', '96%']} w={'100%'} maxW={['100%', '600px']} @@ -168,6 +171,26 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => { )} + {quoteData && ( + + setQuoteData(undefined)} + /> + + )} ); @@ -189,6 +212,7 @@ const Render = (props: Props) => { showRouteToAppDetail={true} showRouteToDatasetDetail={true} isShowReadRawSource={true} + // isShowFullText={true} showNodeStatus > diff --git a/projects/app/src/pageComponents/app/detail/Publish/Link/index.tsx b/projects/app/src/pageComponents/app/detail/Publish/Link/index.tsx index 8280df91f..553e6f50e 100644 --- a/projects/app/src/pageComponents/app/detail/Publish/Link/index.tsx +++ b/projects/app/src/pageComponents/app/detail/Publish/Link/index.tsx @@ -42,7 +42,6 @@ import { getDocPath } from '@/web/common/system/doc'; import dynamic from 'next/dynamic'; import MyMenu from '@fastgpt/web/components/common/MyMenu'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; -import { useI18n } from '@/web/context/I18n'; import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip'; import EmptyTip from '@fastgpt/web/components/common/EmptyTip'; import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel'; @@ -185,6 +184,7 @@ const Share = ({ appId }: { appId: string; type: PublishChannelEnum }) => { name: item.name, responseDetail: item.responseDetail ?? false, showRawSource: item.showRawSource ?? false, + // showFullText: item.showFullText ?? false, showNodeStatus: item.showNodeStatus ?? false, limit: item.limit }) @@ -270,7 +270,6 @@ function EditLinkModal({ }) { const { feConfigs } = useSystemStore(); const { t } = useTranslation(); - const { publishT } = useI18n(); const { register, setValue, @@ -281,6 +280,7 @@ function EditLinkModal({ }); const responseDetail = watch('responseDetail'); + // const showFullText = watch('showFullText'); const showRawSource = watch('showRawSource'); const isEdit = useMemo(() => !!defaultData._id, [defaultData]); @@ -306,7 +306,7 @@ function EditLinkModal({ {t('common:Name')} @@ -353,7 +353,7 @@ function EditLinkModal({ QPM - + @@ -385,11 +385,11 @@ function EditLinkModal({ - {publishT('token_auth')} - + {t('publish:token_auth')} + @@ -400,7 +400,7 @@ function EditLinkModal({ fontSize={'xs'} color={'myGray.500'} > - {publishT('token_auth_use_cases')} + {t('publish:token_auth_use_cases')} )} @@ -421,8 +421,39 @@ function EditLinkModal({ label={t('common:support.outlink.share.Response Quote tips')} > - + + {/* + + {t('common:support.outlink.share.Chat_quote_reader')} + + + + */} {t('common:support.outlink.share.show_complete_quote')} @@ -436,6 +467,7 @@ function EditLinkModal({ onChange(e) { if (e.target.checked) { setValue('responseDetail', true); + // setValue('showFullText', true); } } })} diff --git a/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx b/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx index 49ffeba0c..ed38f89b3 100644 --- a/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx +++ b/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx @@ -7,21 +7,26 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import { useSafeState } from 'ahooks'; import { AppSimpleEditFormType } from '@fastgpt/global/core/app/type'; import { form2AppWorkflow } from '@/web/core/app/utils'; -import { useI18n } from '@/web/context/I18n'; import { useContextSelector } from 'use-context-selector'; import { AppContext } from '../context'; import { useChatTest } from '../useChatTest'; -import ChatItemContextProvider from '@/web/core/chat/context/chatItemContext'; +import ChatItemContextProvider, { ChatItemContext } from '@/web/core/chat/context/chatItemContext'; import ChatRecordContextProvider from '@/web/core/chat/context/chatRecordContext'; import { useChatStore } from '@/web/core/chat/context/useChatStore'; import MyBox from '@fastgpt/web/components/common/MyBox'; +import { cardStyles } from '../constants'; +import ChatQuoteList from '@/pageComponents/chat/ChatQuoteList'; -type Props = { appForm: AppSimpleEditFormType }; -const ChatTest = ({ appForm }: Props) => { +type Props = { + appForm: AppSimpleEditFormType; + setRenderEdit: React.Dispatch>; +}; +const ChatTest = ({ appForm, setRenderEdit }: Props) => { const { t } = useTranslation(); - const { appT } = useI18n(); const { appDetail } = useContextSelector(AppContext, (v) => v); + const quoteData = useContextSelector(ChatItemContext, (v) => v.quoteData); + const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData); const [workflowData, setWorkflowData] = useSafeState({ nodes: appDetail.modules || [], @@ -32,6 +37,11 @@ const ChatTest = ({ appForm }: Props) => { const { nodes, edges } = form2AppWorkflow(appForm, t); setWorkflowData({ nodes, edges }); }, [appForm, setWorkflowData, t]); + + useEffect(() => { + setRenderEdit(!quoteData); + }, [quoteData, setRenderEdit]); + const { ChatContainer, restartChat, loading } = useChatTest({ ...workflowData, chatConfig: appForm.chatConfig, @@ -39,41 +49,56 @@ const ChatTest = ({ appForm }: Props) => { }); return ( - - - - {appT('chat_debug')} + + + + + {t('app:chat_debug')} + + + } + variant={'whiteDanger'} + borderRadius={'md'} + aria-label={'delete'} + onClick={(e) => { + e.stopPropagation(); + restartChat(); + }} + /> + + + + - - } - variant={'whiteDanger'} - borderRadius={'md'} - aria-label={'delete'} - onClick={(e) => { - e.stopPropagation(); - restartChat(); - }} + + {quoteData && ( + + setQuoteData(undefined)} /> - - - - - - + + )} + ); }; -const Render = ({ appForm }: Props) => { +const Render = ({ appForm, setRenderEdit }: Props) => { const { chatId } = useChatStore(); const { appDetail } = useContextSelector(AppContext, (v) => v); @@ -90,10 +115,11 @@ const Render = ({ appForm }: Props) => { showRouteToAppDetail={true} showRouteToDatasetDetail={true} isShowReadRawSource={true} + // isShowFullText={true} showNodeStatus > - + ); diff --git a/projects/app/src/pageComponents/app/detail/SimpleApp/Edit.tsx b/projects/app/src/pageComponents/app/detail/SimpleApp/Edit.tsx index d083e4ed3..85af33adb 100644 --- a/projects/app/src/pageComponents/app/detail/SimpleApp/Edit.tsx +++ b/projects/app/src/pageComponents/app/detail/SimpleApp/Edit.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useState } from 'react'; import { Box } from '@chakra-ui/react'; import ChatTest from './ChatTest'; @@ -21,6 +21,7 @@ const Edit = ({ setPast: (value: React.SetStateAction) => void; }) => { const { isPc } = useSystem(); + const [renderEdit, setRenderEdit] = useState(true); return ( - - - - + {renderEdit && ( + + + + - - + + + - + )} {isPc && ( - - + + )} diff --git a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/ChatTest.tsx b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/ChatTest.tsx index 94c6057a0..4a77f3f23 100644 --- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/ChatTest.tsx +++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/ChatTest.tsx @@ -20,6 +20,7 @@ import ChatRecordContextProvider, { } from '@/web/core/chat/context/chatRecordContext'; import { useChatStore } from '@/web/core/chat/context/useChatStore'; import MyBox from '@fastgpt/web/components/common/MyBox'; +import ChatQuoteList from '@/pageComponents/chat/ChatQuoteList'; type Props = { isOpen: boolean; @@ -41,10 +42,13 @@ const ChatTest = ({ isOpen, nodes = [], edges = [], onClose }: Props) => { }); const pluginRunTab = useContextSelector(ChatItemContext, (v) => v.pluginRunTab); const setPluginRunTab = useContextSelector(ChatItemContext, (v) => v.setPluginRunTab); + const quoteData = useContextSelector(ChatItemContext, (v) => v.quoteData); + const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData); + const chatRecords = useContextSelector(ChatRecordContext, (v) => v.chatRecords); return ( - <> + { left={0} bottom={0} right={0} - onClick={onClose} + onClick={() => { + setQuoteData(undefined); + onClose(); + }} /> { flexDirection={'column'} position={'absolute'} top={5} - right={0} + right={quoteData ? 600 : 0} h={isOpen ? '95%' : '0'} w={isOpen ? ['100%', '460px'] : '0'} bg={'white'} @@ -141,7 +148,27 @@ const ChatTest = ({ isOpen, nodes = [], edges = [], onClose }: Props) => { - + {quoteData && ( + + setQuoteData(undefined)} + /> + + )} + ); }; @@ -162,6 +189,7 @@ const Render = (Props: Props) => { showRouteToAppDetail={true} showRouteToDatasetDetail={true} isShowReadRawSource={true} + // isShowFullText={true} showNodeStatus > diff --git a/projects/app/src/pageComponents/chat/ChatHistorySlider.tsx b/projects/app/src/pageComponents/chat/ChatHistorySlider.tsx index 96fa84926..b8496ede2 100644 --- a/projects/app/src/pageComponents/chat/ChatHistorySlider.tsx +++ b/projects/app/src/pageComponents/chat/ChatHistorySlider.tsx @@ -46,6 +46,7 @@ const ChatHistorySlider = ({ confirmClearText }: { confirmClearText: string }) = const appName = useContextSelector(ChatItemContext, (v) => v.chatBoxData?.app.name); const appAvatar = useContextSelector(ChatItemContext, (v) => v.chatBoxData?.app.avatar); const showRouteToAppDetail = useContextSelector(ChatItemContext, (v) => v.showRouteToAppDetail); + const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData); const concatHistory = useMemo(() => { const formatHistories: HistoryItemType[] = histories.map((item) => { @@ -144,7 +145,10 @@ const ChatHistorySlider = ({ confirmClearText }: { confirmClearText: string }) = borderRadius={'xl'} leftIcon={} overflow={'hidden'} - onClick={() => onChangeChatId()} + onClick={() => { + onChangeChatId(); + setQuoteData(undefined); + }} > {t('common:core.chat.New Chat')} @@ -199,6 +203,7 @@ const ChatHistorySlider = ({ confirmClearText }: { confirmClearText: string }) = : { onClick: () => { onChangeChatId(item.id); + setQuoteData(undefined); } })} {...(i !== concatHistory.length - 1 && { @@ -270,6 +275,7 @@ const ChatHistorySlider = ({ confirmClearText }: { confirmClearText: string }) = onDelHistory(item.id); if (item.id === activeChatId) { onChangeChatId(); + setQuoteData(undefined); } }, type: 'danger' diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx new file mode 100644 index 000000000..4d0285467 --- /dev/null +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx @@ -0,0 +1,192 @@ +import Markdown from '@/components/Markdown'; +import { Box, Flex } from '@chakra-ui/react'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; +import { Dispatch, MutableRefObject, SetStateAction, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { useCopyData } from '@fastgpt/web/hooks/useCopyData'; +import InputDataModal from '@/pageComponents/dataset/detail/InputDataModal'; + +const CollectionQuoteItem = ({ + index, + quoteRefs, + quoteIndex, + setQuoteIndex, + refreshList, + canEdit, + + updated, + isCurrentSelected, + q, + a, + dataId, + collectionId +}: { + index: number; + quoteRefs: MutableRefObject<(HTMLDivElement | null)[]>; + quoteIndex: number; + setQuoteIndex: Dispatch>; + refreshList: () => void; + canEdit: boolean; + + updated?: boolean; + isCurrentSelected: boolean; + q: string; + a?: string; + dataId: string; + collectionId: string; +}) => { + const { t } = useTranslation(); + const { copyData } = useCopyData(); + const hasBeenSearched = quoteIndex !== undefined && quoteIndex > -1; + const [editInputData, setEditInputData] = useState<{ dataId: string; collectionId: string }>(); + + return ( + <> + { + quoteRefs.current[index] = el; + }} + p={2} + py={2} + cursor={hasBeenSearched ? 'pointer' : 'default'} + bg={isCurrentSelected ? '#FFF9E7' : hasBeenSearched ? '#FFFCF2' : ''} + position={'relative'} + overflow={'hidden'} + border={'1px solid '} + borderColor={isCurrentSelected ? 'yellow.200' : 'transparent'} + wordBreak={'break-all'} + fontSize={'sm'} + _hover={ + hasBeenSearched + ? { + '& .hover-data': { visibility: 'visible' } + } + : { + bg: 'linear-gradient(180deg, #FBFBFC 7.61%, #F0F1F6 100%)', + borderTopColor: 'myGray.50', + '& .hover-data': { visibility: 'visible' } + } + } + onClick={(e) => { + e.stopPropagation(); + + if (hasBeenSearched) { + setQuoteIndex(quoteIndex); + } + }} + > + {updated && ( + + + {t('common:core.dataset.data.Updated')} + + )} + + {!!a && ( + + + + )} + + + + + {q.length + (a?.length || 0)} + + + {canEdit && ( + + + setEditInputData({ + dataId, + collectionId + }) + } + > + + + + )} + + { + copyData(q + '\n' + a); + }} + > + + + + + + {editInputData && ( + setEditInputData(undefined)} + onSuccess={() => { + console.log('onSuccess'); + refreshList(); + }} + dataId={editInputData.dataId} + collectionId={editInputData.collectionId} + /> + )} + + ); +}; + +export default CollectionQuoteItem; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx new file mode 100644 index 000000000..53dffec23 --- /dev/null +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx @@ -0,0 +1,345 @@ +import { Box, Button, Flex } from '@chakra-ui/react'; +import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; +import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { useRouter } from 'next/router'; +import { useTranslation } from 'react-i18next'; +import DownloadButton from './DownloadButton'; +import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; +import { downloadFetch } from '@/web/common/system/utils'; +import { useCallback, useEffect, useMemo, useState } from 'react'; +import { getCollectionSource, getDatasetDataPermission } from '@/web/core/dataset/api'; +import { useChatStore } from '@/web/core/chat/context/useChatStore'; +import { useToast } from '@fastgpt/web/hooks/useToast'; +import { getErrText } from '@fastgpt/global/common/error/utils'; +import ScoreTag from './ScoreTag'; +import { formatScore } from '@/components/core/dataset/QuoteItem'; +import NavButton from './NavButton'; +import { useLinkedScroll } from '@fastgpt/web/hooks/useLinkedScroll'; +import CollectionQuoteItem from './CollectionQuoteItem'; +import { DatasetDataListItemType } from '@/global/core/dataset/type'; +import { metadataType } from '@/web/core/chat/context/chatItemContext'; +import { useUserStore } from '@/web/support/user/useUserStore'; +import { getCollectionQuote } from '@/web/core/chat/api'; + +const CollectionReader = ({ + rawSearch, + metadata, + chatTime, + onClose +}: { + rawSearch: SearchDataResponseItemType[]; + metadata: metadataType; + chatTime: Date; + onClose: () => void; +}) => { + const { t } = useTranslation(); + const router = useRouter(); + const { toast } = useToast(); + const { chatId, appId, outLinkAuthData } = useChatStore(); + const { userInfo } = useUserStore(); + const { collectionId, datasetId, chatItemId, sourceId, sourceName } = metadata; + const [quoteIndex, setQuoteIndex] = useState(0); + + const { data: permissionData, loading: isPermissionLoading } = useRequest2( + async () => await getDatasetDataPermission(datasetId), + { + manual: !userInfo && !datasetId, + refreshDeps: [datasetId, userInfo] + } + ); + + const filterResults = useMemo(() => { + const results = rawSearch.filter( + (item) => item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId + ); + + return results.sort((a, b) => (a.chunkIndex || 0) - (b.chunkIndex || 0)); + }, [metadata, rawSearch]); + + const currentQuoteItem = filterResults[quoteIndex]; + + const { + dataList: datasetDataList, + setDataList: setDatasetDataList, + isLoading, + loadData, + ScrollData, + itemRefs, + scrollToItem + } = useLinkedScroll(getCollectionQuote, { + refreshDeps: [collectionId], + params: { + collectionId, + chatTime, + chatItemId, + chatId, + appId, + ...outLinkAuthData + }, + initialId: currentQuoteItem?.id, + initialIndex: currentQuoteItem?.chunkIndex, + canLoadData: !!currentQuoteItem?.id && !isPermissionLoading + }); + + const loading = isLoading || isPermissionLoading; + const isDeleted = !datasetDataList.find((item) => item._id === currentQuoteItem?.id); + + const formatedDataList = useMemo( + () => + datasetDataList.map((item: DatasetDataListItemType) => { + const isCurrentSelected = currentQuoteItem?.id === item._id; + const quoteIndex = filterResults.findIndex((res) => res.id === item._id); + + return { + ...item, + isCurrentSelected, + quoteIndex + }; + }), + [currentQuoteItem?.id, datasetDataList, filterResults] + ); + + useEffect(() => { + setQuoteIndex(0); + setDatasetDataList([]); + }, [collectionId, setDatasetDataList]); + + const { runAsync: handleDownload, loading: downloadLoading } = useRequest2(async () => { + await downloadFetch({ + url: '/api/core/dataset/collection/export', + filename: 'parsed_content.md', + body: { + collectionId: collectionId, + chatTime: chatTime, + chatItemId: chatItemId + } + }); + }); + + const { runAsync: handleRead, loading: readLoading } = useRequest2( + async () => await getCollectionSource({ ...metadata, appId, chatId }), + { + onSuccess: (res) => { + if (!res.value) { + throw new Error('No file found'); + } + if (res.value.startsWith('/')) { + window.open(`${location.origin}${res.value}`, '_blank'); + } else { + window.open(res.value, '_blank'); + } + }, + onError: (err) => { + toast({ + title: t(getErrText(err, t('common:error.fileNotFound'))), + status: 'error' + }); + } + } + ); + + const handleNavigate = useCallback( + async (targetIndex: number) => { + if (targetIndex < 0 || targetIndex >= filterResults.length) return; + const targetItemId = filterResults[targetIndex].id; + const targetItemIndex = filterResults[targetIndex].chunkIndex; + + setQuoteIndex(targetIndex); + const dataIndex = datasetDataList.findIndex((item) => item._id === targetItemId); + + if (dataIndex !== -1) { + setTimeout(() => { + scrollToItem(dataIndex); + }, 50); + } else { + try { + await loadData({ id: targetItemId, index: targetItemIndex }); + } catch (error) { + console.error('Failed to navigate:', error); + } + } + }, + [filterResults, datasetDataList, scrollToItem, loadData] + ); + + return ( + + {/* title */} + + + + + + + {sourceName || t('common:common.UnKnow Source')} + + + + {!!userInfo && permissionData?.permission?.hasReadPer && ( + + )} + + + + + {t('common:core.chat.quote.Quote Tip')} + + + + + + + + {/* header control */} + {datasetDataList.length > 0 && ( + + {/* 引用序号 */} + + + {t('common:core.chat.Quote')} {quoteIndex + 1} + + + / + + + {filterResults.length} + + + + {/* 检索分数 */} + {!loading && + (!isDeleted ? ( + + ) : ( + + + {t('chat:chat.quote.deleted')} + + ))} + + + + {/* 检索按钮 */} + + handleNavigate(quoteIndex - 1)} + /> + handleNavigate(quoteIndex + 1)} + /> + + + )} + + {/* quote list */} + {loading || datasetDataList.length > 0 ? ( + + + {formatedDataList.map((item, index) => ( + } + quoteIndex={item.quoteIndex} + setQuoteIndex={setQuoteIndex} + refreshList={() => + currentQuoteItem?.id && + loadData({ id: currentQuoteItem.id, index: currentQuoteItem.chunkIndex }) + } + updated={item.updated} + isCurrentSelected={item.isCurrentSelected} + q={item.q} + a={item.a} + dataId={item._id} + collectionId={collectionId} + canEdit={!!userInfo && !!permissionData?.permission?.hasWritePer} + /> + ))} + + + ) : ( + + + + + + {t('chat:chat.quote.No Data')} + + + )} + + ); +}; + +export default CollectionReader; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/DownloadButton.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/DownloadButton.tsx new file mode 100644 index 000000000..10fa34424 --- /dev/null +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/DownloadButton.tsx @@ -0,0 +1,68 @@ +import { Button } from '@chakra-ui/react'; +import MyMenu from '@fastgpt/web/components/common/MyMenu'; +import { useTranslation } from 'react-i18next'; +import MyIcon from '@fastgpt/web/components/common/Icon'; + +const DownloadButton = ({ + canAccessRawData, + onDownload, + onRead, + isLoading +}: { + canAccessRawData: boolean; + onDownload: () => void; + onRead: () => void; + isLoading: boolean; +}) => { + const { t } = useTranslation(); + + if (canAccessRawData) { + return ( + } + isLoading={isLoading} + > + {t('common:Download')} + + } + menuList={[ + { + children: [ + { + label: t('common:core.dataset.Download the parsed content'), + type: 'grayBg', + onClick: onDownload + }, + { + label: t('common:core.dataset.Get the raw data'), + type: 'grayBg', + onClick: onRead + } + ] + } + ]} + /> + ); + } + + return ( + + ); +}; + +export default DownloadButton; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/NavButton.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/NavButton.tsx new file mode 100644 index 000000000..a7779ddf4 --- /dev/null +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/NavButton.tsx @@ -0,0 +1,47 @@ +import { Flex } from '@chakra-ui/react'; +import MyIcon from '@fastgpt/web/components/common/Icon'; + +const NavButton = ({ + direction, + isDisabled, + onClick +}: { + direction: 'up' | 'down'; + isDisabled: boolean; + onClick: () => void; +}) => { + const isUp = direction === 'up'; + + const baseStyles = { + color: 'myGray.500', + border: '1px solid', + borderColor: 'myGray.150', + borderRadius: 'sm', + w: 6, + h: 6, + alignItems: 'center', + justifyContent: 'center', + transition: 'all 0.2s' + }; + + const stateStyles = isDisabled + ? { + cursor: 'not-allowed', + opacity: 0.5, + _hover: {} + } + : { + cursor: 'pointer', + opacity: 1, + _hover: { bg: 'myGray.100' }, + onClick + }; + + return ( + + + + ); +}; + +export default NavButton; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/QuoteItem.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/QuoteItem.tsx new file mode 100644 index 000000000..6f2730bfc --- /dev/null +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/QuoteItem.tsx @@ -0,0 +1,163 @@ +import { ScoreItemType } from '@/components/core/dataset/QuoteItem'; +import { Box, Flex } from '@chakra-ui/react'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import ScoreTag from './ScoreTag'; +import Markdown from '@/components/Markdown'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; +import { useTranslation } from 'react-i18next'; +import { useCopyData } from '@fastgpt/web/hooks/useCopyData'; + +const QuoteItem = ({ + index, + icon, + sourceName, + score, + q, + a +}: { + index: number; + icon: string; + sourceName: string; + score: { primaryScore?: ScoreItemType; secondaryScore: ScoreItemType[] }; + q: string; + a?: string; +}) => { + const { t } = useTranslation(); + const { copyData } = useCopyData(); + const isDeleted = !q; + + return ( + + + + + {index + 1} + + + + + {sourceName} + + + + {score && !isDeleted && ( + + + + )} + + {!isDeleted ? ( + <> + + {!!a && ( + + + + )} + + ) : ( + + + {t('chat:chat.quote.deleted')} + + )} + + + + + {q.length + (a?.length || 0)} + + + + { + copyData(q + '\n' + a); + }} + > + + + + + + ); +}; + +export default QuoteItem; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/QuoteReader.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/QuoteReader.tsx new file mode 100644 index 000000000..84fe881e6 --- /dev/null +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/QuoteReader.tsx @@ -0,0 +1,152 @@ +import { Box, Flex } from '@chakra-ui/react'; +import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import MyBox from '@fastgpt/web/components/common/MyBox'; +import { useTranslation } from 'react-i18next'; +import { useRequest2 } from '@fastgpt/web/hooks/useRequest'; +import { useChatStore } from '@/web/core/chat/context/useChatStore'; +import QuoteItem from './QuoteItem'; +import { useMemo } from 'react'; +import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; +import { formatScore } from '@/components/core/dataset/QuoteItem'; +import { metadataType } from '@/web/core/chat/context/chatItemContext'; +import { getQuoteDataList } from '@/web/core/chat/api'; + +const QuoteReader = ({ + rawSearch, + metadata, + chatTime, + onClose +}: { + rawSearch: SearchDataResponseItemType[]; + metadata: metadataType; + chatTime: Date; + onClose: () => void; +}) => { + const { t } = useTranslation(); + + const { chatId, appId, outLinkAuthData } = useChatStore(); + + const { data, loading } = useRequest2( + async () => + await getQuoteDataList({ + datasetDataIdList: rawSearch.map((item) => item.id), + chatTime, + collectionIdList: metadata.collectionIdList, + chatItemId: metadata.chatItemId, + appId, + chatId, + ...outLinkAuthData + }), + { + manual: false + } + ); + + const filterResults = useMemo(() => { + if (!metadata.collectionId) { + return rawSearch; + } + + return rawSearch.filter( + (item) => item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId + ); + }, [metadata, rawSearch]); + + const formatedDataList = useMemo(() => { + return filterResults + .map((item) => { + const currentFilterItem = data?.quoteList.find((res) => res._id === item.id); + + return { + ...item, + q: currentFilterItem?.q || '', + a: currentFilterItem?.a || '', + score: formatScore(item.score), + icon: getSourceNameIcon({ + sourceId: item.sourceId, + sourceName: item.sourceName + }) + }; + }) + .sort((a, b) => { + return (b.score.primaryScore?.value || 0) - (a.score.primaryScore?.value || 0); + }); + }, [data?.quoteList, filterResults]); + + return ( + + {/* title */} + + + + + + {metadata.sourceName + ? metadata.sourceName + : t('common:core.chat.Quote Amount', { amount: rawSearch.length })} + + + + {t('common:core.chat.quote.Quote Tip')} + + + + + + + + {/* quote list */} + + {!loading && ( + + {formatedDataList?.map((item, index) => ( + + ))} + + )} + + + ); +}; + +export default QuoteReader; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/ScoreTag.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/ScoreTag.tsx new file mode 100644 index 000000000..16b8a00d5 --- /dev/null +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/ScoreTag.tsx @@ -0,0 +1,78 @@ +import { ScoreItemType, scoreTheme } from '@/components/core/dataset/QuoteItem'; +import { Box, Flex, Progress } from '@chakra-ui/react'; +import { SearchScoreTypeMap } from '@fastgpt/global/core/dataset/constants'; +import MyTooltip from '@fastgpt/web/components/common/MyTooltip'; +import { useTranslation } from 'react-i18next'; + +const ScoreTag = (score: { primaryScore?: ScoreItemType; secondaryScore: ScoreItemType[] }) => { + const { t } = useTranslation(); + + return ( + + {score?.primaryScore && ( + + {score.secondaryScore.map((item, i) => ( + + + + #{item.index + 1} + + + {t(SearchScoreTypeMap[item.type]?.label as any)}: {item.value.toFixed(4)} + + + + {SearchScoreTypeMap[item.type]?.showScore && ( + + )} + + + ))} + + ) : ( + t(SearchScoreTypeMap[score.primaryScore.type]?.desc as any) + ) + } + > + + + {t(SearchScoreTypeMap[score.primaryScore.type]?.label as any)} + {SearchScoreTypeMap[score.primaryScore.type]?.showScore + ? ` ${score.primaryScore.value.toFixed(4)}` + : `: ${score.primaryScore.index + 1}`} + + + + )} + + ); +}; + +export default ScoreTag; diff --git a/projects/app/src/pageComponents/chat/ChatQuoteList/index.tsx b/projects/app/src/pageComponents/chat/ChatQuoteList/index.tsx new file mode 100644 index 000000000..233900635 --- /dev/null +++ b/projects/app/src/pageComponents/chat/ChatQuoteList/index.tsx @@ -0,0 +1,42 @@ +import React from 'react'; +import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; +import { useContextSelector } from 'use-context-selector'; +import { ChatItemContext, metadataType } from '@/web/core/chat/context/chatItemContext'; +import CollectionQuoteReader from './CollectionQuoteReader'; +import QuoteReader from './QuoteReader'; + +const ChatQuoteList = ({ + chatTime, + rawSearch = [], + metadata, + onClose +}: { + chatTime: Date; + rawSearch: SearchDataResponseItemType[]; + metadata: metadataType; + onClose: () => void; +}) => { + const isShowReadRawSource = useContextSelector(ChatItemContext, (v) => v.isShowReadRawSource); + + return ( + <> + {metadata.collectionId && isShowReadRawSource ? ( + + ) : ( + + )} + + ); +}; + +export default ChatQuoteList; diff --git a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx index 1b7046500..4dc843a29 100644 --- a/projects/app/src/pageComponents/dataset/detail/DataCard.tsx +++ b/projects/app/src/pageComponents/dataset/detail/DataCard.tsx @@ -5,7 +5,6 @@ import { delOneDatasetDataById, getDatasetCollectionById } from '@/web/core/dataset/api'; -import { useQuery } from '@tanstack/react-query'; import { useToast } from '@fastgpt/web/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { useConfirm } from '@fastgpt/web/hooks/useConfirm'; diff --git a/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx b/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx index bbb9a73a0..f3397cdf8 100644 --- a/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx +++ b/projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx @@ -178,7 +178,6 @@ const InputDataModal = ({ async (e: InputDataType) => { if (!dataId) return Promise.reject(t('common:common.error.unKnow')); - // not exactly same await putDatasetDataById({ dataId, q: e.q, diff --git a/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts b/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts new file mode 100644 index 000000000..99ffc0d51 --- /dev/null +++ b/projects/app/src/pages/api/core/chat/quote/getCollectionQuote.ts @@ -0,0 +1,250 @@ +import { NextAPI } from '@/service/middleware/entry'; +import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat'; +import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { ApiRequestProps } from '@fastgpt/service/type/next'; +import { LinkedListResponse, LinkedPaginationProps } from '@fastgpt/web/common/fetch/type'; +import { FilterQuery, Types } from 'mongoose'; +import { dataFieldSelector, processChatTimeFilter } from './getQuote'; + +export type GetCollectionQuoteProps = LinkedPaginationProps & { + chatTime: Date; + + isInitialLoad: boolean; + collectionId: string; + chatItemId: string; + appId: string; + chatId: string; + shareId?: string; + outLinkUid?: string; + teamId?: string; + teamToken?: string; +}; + +export type GetCollectionQuoteRes = LinkedListResponse; + +type BaseMatchType = FilterQuery; + +async function handler( + req: ApiRequestProps +): Promise { + const { + initialId, + initialIndex, + prevId, + prevIndex, + nextId, + nextIndex, + chatTime, + + isInitialLoad, + collectionId, + chatItemId, + appId, + chatId, + shareId, + outLinkUid, + teamId, + teamToken, + pageSize = 15 + } = req.body; + const limitedPageSize = Math.min(pageSize, 30); + + await Promise.all([ + authChatCrud({ + req, + authToken: true, + appId, + chatId, + shareId, + outLinkUid, + teamId, + teamToken + }), + authCollectionInChat({ appId, chatId, chatItemId, collectionId }) + ]); + + const baseMatch: BaseMatchType = { + collectionId, + $or: [ + { updateTime: { $lt: new Date(chatTime) } }, + { history: { $elemMatch: { updateTime: { $lt: new Date(chatTime) } } } } + ] + }; + + if (initialId && initialIndex !== undefined) { + return await handleInitialLoad( + initialId, + initialIndex, + limitedPageSize, + chatTime, + chatItemId, + isInitialLoad, + baseMatch + ); + } + + if ((prevId && prevIndex !== undefined) || (nextId && nextIndex !== undefined)) { + return await handlePaginatedLoad( + prevId, + prevIndex, + nextId, + nextIndex, + limitedPageSize, + chatTime, + chatItemId, + baseMatch + ); + } + + return { list: [], hasMorePrev: false, hasMoreNext: false }; +} + +export default NextAPI(handler); + +async function handleInitialLoad( + initialId: string, + initialIndex: number, + pageSize: number, + chatTime: Date, + chatItemId: string, + isInitialLoad: boolean, + baseMatch: BaseMatchType +): Promise { + const centerNode = await MongoDatasetData.findOne( + { + _id: new Types.ObjectId(initialId) + }, + dataFieldSelector + ).lean(); + + if (!centerNode) { + if (isInitialLoad) { + const list = await MongoDatasetData.find(baseMatch, dataFieldSelector) + .sort({ chunkIndex: 1, _id: -1 }) + .limit(pageSize) + .lean(); + + const listRes = list.map((item, index) => ({ + ...item, + index: item.chunkIndex + })); + + const hasMoreNext = list.length === pageSize; + + return { + list: listRes, + hasMorePrev: false, + hasMoreNext + }; + } + + return Promise.reject('centerNode not found'); + } + + const prevHalfSize = Math.floor(pageSize / 2); + const nextHalfSize = pageSize - prevHalfSize - 1; + + const { list: prevList, hasMore: hasMorePrev } = await getPrevNodes( + initialId, + initialIndex, + prevHalfSize, + baseMatch + ); + const { list: nextList, hasMore: hasMoreNext } = await getNextNodes( + initialId, + initialIndex, + nextHalfSize, + baseMatch + ); + + const resultList = [...prevList, centerNode, ...nextList]; + + const list = processChatTimeFilter(resultList, chatTime); + + return { + list: list.map((item) => ({ + ...item, + index: item.chunkIndex + })), + hasMorePrev, + hasMoreNext + }; +} + +async function handlePaginatedLoad( + prevId: string | undefined, + prevIndex: number | undefined, + nextId: string | undefined, + nextIndex: number | undefined, + pageSize: number, + chatTime: Date, + chatItemId: string, + baseMatch: BaseMatchType +): Promise { + const { list, hasMore } = + prevId && prevIndex !== undefined + ? await getPrevNodes(prevId, prevIndex, pageSize, baseMatch) + : await getNextNodes(nextId!, nextIndex!, pageSize, baseMatch); + + const processedList = processChatTimeFilter(list, chatTime); + + return { + list: processedList.map((item) => ({ + ...item, + index: item.chunkIndex + })), + hasMorePrev: !!prevId && hasMore, + hasMoreNext: !!nextId && hasMore + }; +} + +async function getPrevNodes( + initialId: string, + initialIndex: number, + limit: number, + baseMatch: BaseMatchType +) { + const match: BaseMatchType = { + ...baseMatch, + $or: [ + { chunkIndex: { $lte: initialIndex } }, + { chunkIndex: initialIndex, _id: { $lte: new Types.ObjectId(initialId) } } + ] + }; + + const list = await MongoDatasetData.find(match, dataFieldSelector) + .sort({ chunkIndex: -1, _id: 1 }) + .limit(limit) + .lean(); + + return { + list: list.filter((item) => String(item._id) !== initialId).reverse(), + hasMore: list.length === limit + }; +} + +async function getNextNodes( + initialId: string, + initialIndex: number, + limit: number, + baseMatch: BaseMatchType +) { + const match: BaseMatchType = { + ...baseMatch, + $or: [ + { chunkIndex: { $gte: initialIndex } }, + { chunkIndex: initialIndex, _id: { $gte: new Types.ObjectId(initialId) } } + ] + }; + + const list = await MongoDatasetData.find(match, dataFieldSelector) + .sort({ chunkIndex: 1, _id: -1 }) + .limit(limit) + .lean(); + + return { + list: list.filter((item) => String(item._id) !== initialId), + hasMore: list.length === limit + }; +} diff --git a/projects/app/src/pages/api/core/chat/quote/getQuote.ts b/projects/app/src/pages/api/core/chat/quote/getQuote.ts new file mode 100644 index 000000000..c6e35f96f --- /dev/null +++ b/projects/app/src/pages/api/core/chat/quote/getQuote.ts @@ -0,0 +1,102 @@ +import { NextAPI } from '@/service/middleware/entry'; +import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat'; +import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { ApiRequestProps } from '@fastgpt/service/type/next'; + +export type GetQuoteDataProps = { + datasetDataIdList: string[]; + chatTime: Date; + + collectionIdList: string[]; + chatItemId: string; + appId: string; + chatId: string; + shareId?: string; + outLinkUid?: string; + teamId?: string; + teamToken?: string; +}; + +export type GetQuoteDataRes = { + quoteList: DatasetDataSchemaType[]; +}; + +export const dataFieldSelector = + '_id datasetId collectionId q a chunkIndex history updateTime currentChatItemId prevId'; + +async function handler(req: ApiRequestProps): Promise { + const { + datasetDataIdList, + chatTime, + + collectionIdList, + chatItemId, + chatId, + appId, + shareId, + outLinkUid, + teamId, + teamToken + } = req.body; + + await authChatCrud({ + req, + authToken: true, + appId, + chatId, + shareId, + outLinkUid, + teamId, + teamToken + }); + + await Promise.all( + collectionIdList.map(async (collectionId) => { + await authCollectionInChat({ appId, chatId, chatItemId, collectionId }); + }) + ); + + const list = await MongoDatasetData.find( + { _id: { $in: datasetDataIdList } }, + dataFieldSelector + ).lean(); + + const quoteList = processChatTimeFilter(list, chatTime); + + return { + quoteList + }; +} + +export default NextAPI(handler); + +export function processChatTimeFilter(list: DatasetDataSchemaType[], chatTime?: Date) { + if (!chatTime) return list; + + return list.map((item) => { + if (!item.history) return item; + + const { history, ...rest } = item; + const formatedChatTime = new Date(chatTime); + + if (item.updateTime <= formatedChatTime) { + return rest; + } + + const latestHistoryIndex = history.findIndex( + (historyItem: any) => historyItem.updateTime <= formatedChatTime + ); + + if (latestHistoryIndex === -1) return rest; + + const latestHistory = history[latestHistoryIndex]; + + return { + ...rest, + q: latestHistory?.q || item.q, + a: latestHistory?.a || item.a, + updated: true + }; + }); +} diff --git a/projects/app/src/pages/api/core/dataset/collection/export.ts b/projects/app/src/pages/api/core/dataset/collection/export.ts new file mode 100644 index 000000000..d9111e806 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/export.ts @@ -0,0 +1,78 @@ +import { NextAPI } from '@/service/middleware/entry'; +import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; +import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit'; +import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; +import { responseWriteController } from '@fastgpt/service/common/response'; +import { addLog } from '@fastgpt/service/common/system/log'; +import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; +import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth'; +import { ApiRequestProps } from '@fastgpt/service/type/next'; +import { NextApiResponse } from 'next'; + +export type ExportCollectionBody = { + collectionId: string; + chatTime: Date; +}; + +async function handler(req: ApiRequestProps, res: NextApiResponse) { + let { collectionId, chatTime } = req.body; + + const { teamId, collection } = await authDatasetCollection({ + req, + authToken: true, + collectionId, + per: ReadPermissionVal + }); + + const where = { + teamId, + datasetId: collection.datasetId, + collectionId, + ...(chatTime + ? { + $or: [ + { updateTime: { $lt: new Date(chatTime) } }, + { history: { $elemMatch: { updateTime: { $lt: new Date(chatTime) } } } } + ] + } + : {}) + }; + + res.setHeader('Content-Type', 'text/csv; charset=utf-8;'); + res.setHeader('Content-Disposition', 'attachment; filename=usage.csv; '); + + const cursor = MongoDatasetData.find(where, 'q a', { + ...readFromSecondary, + batchSize: 1000 + }) + .sort({ chunkIndex: 1 }) + .limit(50000) + .cursor(); + + const write = responseWriteController({ + res, + readStream: cursor + }); + + cursor.on('data', (doc) => { + const res = doc.a ? `\n${doc.q}\n${doc.a}` : `\n${doc.q}`; + + write(res); + }); + + cursor.on('end', () => { + cursor.close(); + res.end(); + }); + + cursor.on('error', (err) => { + addLog.error(`export usage error`, err); + res.status(500); + res.end(); + }); +} + +export default NextAPI( + useIPFrequencyLimit({ id: 'export-usage', seconds: 60, limit: 1, force: true }), + handler +); diff --git a/projects/app/src/pages/api/core/dataset/collection/read.ts b/projects/app/src/pages/api/core/dataset/collection/read.ts index de9b1dbb2..17bb7141d 100644 --- a/projects/app/src/pages/api/core/dataset/collection/read.ts +++ b/projects/app/src/pages/api/core/dataset/collection/read.ts @@ -7,9 +7,7 @@ import { BucketNameEnum, ReadFileBaseUrl } from '@fastgpt/global/common/file/con import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; -import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema'; -import { AIChatItemType, ChatHistoryItemResType } from '@fastgpt/global/core/chat/type'; -import { authChatCrud } from '@/service/support/permission/auth/chat'; +import { authChatCrud, authCollectionInChat } from '@/service/support/permission/auth/chat'; import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller'; import { useApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset/api'; import { POST } from '@fastgpt/service/common/api/plusRequest'; @@ -29,57 +27,6 @@ export type readCollectionSourceResponse = { value: string; }; -const authCollectionInChat = async ({ - collectionId, - appId, - chatId, - chatItemId -}: { - collectionId: string; - appId: string; - chatId: string; - chatItemId: string; -}) => { - try { - const chatItem = (await MongoChatItem.findOne( - { - appId, - chatId, - dataId: chatItemId - }, - 'responseData' - ).lean()) as AIChatItemType; - - if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetFile); - - // 找 responseData 里，是否有该文档 id - const responseData = chatItem.responseData || []; - const flatResData: ChatHistoryItemResType[] = - responseData - ?.map((item) => { - return [ - item, - ...(item.pluginDetail || []), - ...(item.toolDetail || []), - ...(item.loopDetail || []) - ]; - }) - .flat() || []; - - if ( - flatResData.some((item) => { - if (item.quoteList) { - return item.quoteList.some((quote) => quote.collectionId === collectionId); - } - return false; - }) - ) { - return true; - } - } catch (error) {} - return Promise.reject(DatasetErrEnum.unAuthDatasetFile); -}; - async function handler( req: ApiRequestProps ): Promise { diff --git a/projects/app/src/pages/api/core/dataset/data/getPermission.ts b/projects/app/src/pages/api/core/dataset/data/getPermission.ts new file mode 100644 index 000000000..16191e9ec --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/data/getPermission.ts @@ -0,0 +1,53 @@ +import type { NextApiRequest } from 'next'; +import { NextAPI } from '@/service/middleware/entry'; +import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; +import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; +import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; + +export type GetQuotePermissionResponse = + | { + permission: { + hasWritePer: boolean; + hasReadPer: boolean; + }; + } + | undefined; + +async function handler(req: NextApiRequest): Promise { + const { id: datasetId } = req.query as { + id?: string; + }; + if (!datasetId) { + return Promise.reject('datasetId is required'); + } + + try { + const { permission } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: ReadPermissionVal + }); + + return { + permission: { + hasReadPer: permission.hasReadPer, + hasWritePer: permission.hasWritePer + } + }; + } catch (error) { + if (error === DatasetErrEnum.unAuthDataset) { + return { + permission: { + hasWritePer: false, + hasReadPer: false + } + }; + } + + return Promise.reject(error); + } +} + +export default NextAPI(handler); diff --git a/projects/app/src/pages/api/core/dataset/data/v2/list.ts b/projects/app/src/pages/api/core/dataset/data/v2/list.ts index 0caa766d9..1980e6e8c 100644 --- a/projects/app/src/pages/api/core/dataset/data/v2/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/v2/list.ts @@ -45,7 +45,7 @@ async function handler( const [list, total] = await Promise.all([ MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex') - .sort({ chunkIndex: 1, updateTime: -1 }) + .sort({ chunkIndex: 1, _id: -1 }) .skip(offset) .limit(pageSize) .lean(), diff --git a/projects/app/src/pages/api/support/outLink/update.ts b/projects/app/src/pages/api/support/outLink/update.ts index ba32f8ba5..01aecef5a 100644 --- a/projects/app/src/pages/api/support/outLink/update.ts +++ b/projects/app/src/pages/api/support/outLink/update.ts @@ -37,6 +37,7 @@ async function handler( responseDetail, showRawSource, showNodeStatus, + // showFullText, limit, app }); diff --git a/projects/app/src/pages/api/v1/chat/completions.ts b/projects/app/src/pages/api/v1/chat/completions.ts index 5657a2783..14f79d53f 100644 --- a/projects/app/src/pages/api/v1/chat/completions.ts +++ b/projects/app/src/pages/api/v1/chat/completions.ts @@ -59,6 +59,7 @@ import { getWorkflowResponseWrite } from '@fastgpt/service/core/workflow/dispatc import { WORKFLOW_MAX_RUN_TIMES } from '@fastgpt/service/core/workflow/constants'; import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils'; import { ExternalProviderType } from '@fastgpt/global/core/workflow/runtime/type'; +import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant'; type FastGptWebChatProps = { chatId?: string; // undefined: get histories from messages, '': new chat, 'xxxxx': get histories from db diff --git a/projects/app/src/pages/chat/index.tsx b/projects/app/src/pages/chat/index.tsx index f9339b1d2..0802e50d0 100644 --- a/projects/app/src/pages/chat/index.tsx +++ b/projects/app/src/pages/chat/index.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useMemo } from 'react'; +import React, { useCallback, useEffect, useMemo, useState, useRef } from 'react'; import NextHead from '@/components/common/NextHead'; import { useRouter } from 'next/router'; import { getInitChatInfo } from '@/web/core/chat/api'; @@ -36,6 +36,7 @@ import ChatItemContextProvider, { ChatItemContext } from '@/web/core/chat/contex import ChatRecordContextProvider, { ChatRecordContext } from '@/web/core/chat/context/chatRecordContext'; +import ChatQuoteList from '@/pageComponents/chat/ChatQuoteList'; const CustomPluginRunBox = dynamic(() => import('@/pageComponents/chat/CustomPluginRunBox')); @@ -58,6 +59,8 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => { const isPlugin = useContextSelector(ChatItemContext, (v) => v.isPlugin); const chatBoxData = useContextSelector(ChatItemContext, (v) => v.chatBoxData); const setChatBoxData = useContextSelector(ChatItemContext, (v) => v.setChatBoxData); + const quoteData = useContextSelector(ChatItemContext, (v) => v.quoteData); + const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData); const chatRecords = useContextSelector(ChatRecordContext, (v) => v.chatRecords); const totalRecordsCount = useContextSelector(ChatRecordContext, (v) => v.totalRecordsCount); @@ -138,13 +141,14 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => { }, [appId, chatId, onUpdateHistoryTitle, setChatBoxData, forbidLoadChat] ); + const RenderHistorySlider = useMemo(() => { const Children = ( ); return isPc || !appId ? ( - {Children} + {Children} ) : ( { {Children} ); - }, [appId, isOpenSlider, isPc, onCloseSlider, t]); + }, [t, isPc, appId, isOpenSlider, onCloseSlider, quoteData]); return ( @@ -169,7 +173,14 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => { )} - + {/* pc always show history. */} {RenderHistorySlider} @@ -215,6 +226,16 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => { + {quoteData && ( + + setQuoteData(undefined)} + /> + + )} ); }; @@ -278,6 +299,7 @@ const Render = (props: { appId: string; isStandalone?: string }) => { showRouteToAppDetail={isStandalone !== '1'} showRouteToDatasetDetail={isStandalone !== '1'} isShowReadRawSource={true} + // isShowFullText={true} showNodeStatus > diff --git a/projects/app/src/pages/chat/share.tsx b/projects/app/src/pages/chat/share.tsx index 9fc6baa4f..9f114147d 100644 --- a/projects/app/src/pages/chat/share.tsx +++ b/projects/app/src/pages/chat/share.tsx @@ -37,6 +37,7 @@ import { useChatStore } from '@/web/core/chat/context/useChatStore'; import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants'; import { useI18nLng } from '@fastgpt/web/hooks/useI18n'; import { AppSchema } from '@fastgpt/global/core/app/type'; +import ChatQuoteList from '@/pageComponents/chat/ChatQuoteList'; const CustomPluginRunBox = dynamic(() => import('@/pageComponents/chat/CustomPluginRunBox')); @@ -49,6 +50,7 @@ type Props = { authToken: string; customUid: string; showRawSource: boolean; + // showFullText: boolean; showNodeStatus: boolean; }; @@ -81,6 +83,8 @@ const OutLink = (props: Props) => { const resetVariables = useContextSelector(ChatItemContext, (v) => v.resetVariables); const isPlugin = useContextSelector(ChatItemContext, (v) => v.isPlugin); const setChatBoxData = useContextSelector(ChatItemContext, (v) => v.setChatBoxData); + const quoteData = useContextSelector(ChatItemContext, (v) => v.quoteData); + const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData); const chatRecords = useContextSelector(ChatRecordContext, (v) => v.chatRecords); const totalRecordsCount = useContextSelector(ChatRecordContext, (v) => v.totalRecordsCount); @@ -217,7 +221,7 @@ const OutLink = (props: Props) => { if (showHistory !== '1') return null; return isPc ? ( - {Children} + {Children} ) : ( { ); - }, [isOpenSlider, isPc, onCloseSlider, showHistory, t]); + }, [isOpenSlider, isPc, onCloseSlider, quoteData, showHistory, t]); return ( - <> + { - + {quoteData && ( + + setQuoteData(undefined)} + /> + + )} + ); }; @@ -340,6 +354,7 @@ const Render = (props: Props) => { showRouteToAppDetail={false} showRouteToDatasetDetail={false} isShowReadRawSource={props.showRawSource} + // isShowFullText={props.showFullText} showNodeStatus={props.showNodeStatus} > @@ -383,6 +398,7 @@ export async function getServerSideProps(context: any) { appAvatar: app?.associatedApp?.avatar ?? '', appIntro: app?.associatedApp?.intro ?? 'AI', showRawSource: app?.showRawSource ?? false, + // showFullText: app?.showFullText ?? false, showNodeStatus: app?.showNodeStatus ?? false, shareId: shareId ?? '', authToken: authToken ?? '', diff --git a/projects/app/src/pages/chat/team.tsx b/projects/app/src/pages/chat/team.tsx index cc0b94097..dd02531e7 100644 --- a/projects/app/src/pages/chat/team.tsx +++ b/projects/app/src/pages/chat/team.tsx @@ -33,6 +33,7 @@ import ChatRecordContextProvider, { import { useChatStore } from '@/web/core/chat/context/useChatStore'; import { useMount } from 'ahooks'; import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants'; +import ChatQuoteList from '@/pageComponents/chat/ChatQuoteList'; const CustomPluginRunBox = dynamic(() => import('@/pageComponents/chat/CustomPluginRunBox')); type Props = { appId: string; chatId: string; teamId: string; teamToken: string }; @@ -63,6 +64,8 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => { const resetVariables = useContextSelector(ChatItemContext, (v) => v.resetVariables); const chatBoxData = useContextSelector(ChatItemContext, (v) => v.chatBoxData); const setChatBoxData = useContextSelector(ChatItemContext, (v) => v.setChatBoxData); + const quoteData = useContextSelector(ChatItemContext, (v) => v.quoteData); + const setQuoteData = useContextSelector(ChatItemContext, (v) => v.setQuoteData); const chatRecords = useContextSelector(ChatRecordContext, (v) => v.chatRecords); const totalRecordsCount = useContextSelector(ChatRecordContext, (v) => v.totalRecordsCount); @@ -163,7 +166,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => { ); return isPc || !appId ? ( - {Children} + {Children} ) : ( { {Children} ); - }, [appId, isOpenSlider, isPc, onCloseSlider, t]); + }, [appId, isOpenSlider, isPc, onCloseSlider, quoteData, t]); return ( @@ -231,6 +234,17 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => { + + {quoteData && ( + + setQuoteData(undefined)} + /> + + )} ); }; @@ -300,6 +314,7 @@ const Render = (props: Props) => { showRouteToAppDetail={false} showRouteToDatasetDetail={false} isShowReadRawSource={true} + // isShowFullText={true} showNodeStatus > diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts index ae77ce77e..1c794b1c7 100644 --- a/projects/app/src/service/core/dataset/data/controller.ts +++ b/projects/app/src/service/core/dataset/data/controller.ts @@ -220,11 +220,7 @@ export async function updateData2Dataset({ } } - // 4. Update mongo updateTime(便于脏数据检查器识别) - mongoData.updateTime = new Date(); - await mongoData.save(); - - // 5. Insert vector + // insert vector const insertResult = await Promise.all( patchResult .filter((item) => item.type === 'create' || item.type === 'update') @@ -249,9 +245,19 @@ export async function updateData2Dataset({ .filter((item) => item.type !== 'delete') .map((item) => item.index) as DatasetDataIndexItemType[]; - // console.log(clonePatchResult2Insert); await mongoSessionRun(async (session) => { - // Update MongoData + // update mongo data + mongoData.history = + q !== mongoData.q || a !== mongoData.a + ? [ + { + q: mongoData.q, + a: mongoData.a, + updateTime: mongoData.updateTime + }, + ...(mongoData.history?.slice(0, 9) || []) + ] + : mongoData.history; mongoData.q = q || mongoData.q; mongoData.a = a ?? mongoData.a; mongoData.indexes = newIndexes; @@ -277,6 +283,10 @@ export async function updateData2Dataset({ } }); + // Update mongo updateTime(便于脏数据检查器识别) + mongoData.updateTime = new Date(); + await mongoData.save(); + return { tokens }; diff --git a/projects/app/src/service/support/permission/auth/chat.ts b/projects/app/src/service/support/permission/auth/chat.ts index 121d10742..6a9b21a24 100644 --- a/projects/app/src/service/support/permission/auth/chat.ts +++ b/projects/app/src/service/support/permission/auth/chat.ts @@ -1,4 +1,4 @@ -import { ChatSchema } from '@fastgpt/global/core/chat/type'; +import { AIChatItemType, ChatHistoryItemResType, ChatSchema } from '@fastgpt/global/core/chat/type'; import { MongoChat } from '@fastgpt/service/core/chat/chatSchema'; import { AuthModeType } from '@fastgpt/service/support/permission/type'; import { authOutLink } from './outLink'; @@ -6,6 +6,8 @@ import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat'; import { authTeamSpaceToken } from './team'; import { AuthUserTypeEnum, ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { authApp } from '@fastgpt/service/support/permission/app/auth'; +import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema'; +import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; /* 检查chat的权限： @@ -184,3 +186,54 @@ export async function authChatCrud({ return Promise.reject(ChatErrEnum.unAuthChat); } + +export const authCollectionInChat = async ({ + collectionId, + appId, + chatId, + chatItemId +}: { + collectionId: string; + appId: string; + chatId: string; + chatItemId: string; +}) => { + try { + const chatItem = (await MongoChatItem.findOne( + { + appId, + chatId, + dataId: chatItemId + }, + 'responseData' + ).lean()) as AIChatItemType; + + if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetFile); + + // 找 responseData 里，是否有该文档 id + const responseData = chatItem.responseData || []; + const flatResData: ChatHistoryItemResType[] = + responseData + ?.map((item) => { + return [ + item, + ...(item.pluginDetail || []), + ...(item.toolDetail || []), + ...(item.loopDetail || []) + ]; + }) + .flat() || []; + + if ( + flatResData.some((item) => { + if (item.quoteList) { + return item.quoteList.some((quote) => quote.collectionId === collectionId); + } + return false; + }) + ) { + return true; + } + } catch (error) {} + return Promise.reject(DatasetErrEnum.unAuthDatasetFile); +}; diff --git a/projects/app/src/web/core/app/constants.ts b/projects/app/src/web/core/app/constants.ts index a5aa10fda..d60d2b72d 100644 --- a/projects/app/src/web/core/app/constants.ts +++ b/projects/app/src/web/core/app/constants.ts @@ -28,6 +28,7 @@ export const defaultOutLinkForm: OutLinkEditType = { name: '', showNodeStatus: true, responseDetail: false, + // showFullText: false, showRawSource: false, limit: { QPM: 100, diff --git a/projects/app/src/web/core/chat/api.ts b/projects/app/src/web/core/chat/api.ts index f4e49e351..e9bc72bdc 100644 --- a/projects/app/src/web/core/chat/api.ts +++ b/projects/app/src/web/core/chat/api.ts @@ -30,6 +30,11 @@ import type { getPaginationRecordsBody, getPaginationRecordsResponse } from '@/pages/api/core/chat/getPaginationRecords'; +import { GetQuoteDataProps, GetQuoteDataRes } from '@/pages/api/core/chat/quote/getQuote'; +import { + GetCollectionQuoteProps, + GetCollectionQuoteRes +} from '@/pages/api/core/chat/quote/getCollectionQuote'; /** * 获取初始化聊天内容 @@ -100,3 +105,9 @@ export const getMyTokensApps = (data: AuthTeamTagTokenProps) => */ export const getinitTeamChat = (data: { teamId: string; authToken: string; appId: string }) => GET(`/proApi/core/chat/initTeamChat`, data); + +export const getQuoteDataList = (data: GetQuoteDataProps) => + POST(`/core/chat/quote/getQuote`, data); + +export const getCollectionQuote = (data: GetCollectionQuoteProps) => + POST(`/core/chat/quote/getCollectionQuote`, data); diff --git a/projects/app/src/web/core/chat/context/chatItemContext.tsx b/projects/app/src/web/core/chat/context/chatItemContext.tsx index 633fae4d2..6a2cd9437 100644 --- a/projects/app/src/web/core/chat/context/chatItemContext.tsx +++ b/projects/app/src/web/core/chat/context/chatItemContext.tsx @@ -1,6 +1,6 @@ import { ChatBoxInputFormType } from '@/components/core/chat/ChatContainer/ChatBox/type'; import { PluginRunBoxTabEnum } from '@/components/core/chat/ChatContainer/PluginRunBox/constants'; -import React, { ReactNode, useCallback, useMemo, useRef, useState } from 'react'; +import React, { ReactNode, useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { createContext } from 'use-context-selector'; import { ComponentRef as ChatComponentRef } from '@/components/core/chat/ChatContainer/ChatBox/type'; import { useForm, UseFormReturn } from 'react-hook-form'; @@ -8,11 +8,13 @@ import { defaultChatData } from '@/global/core/chat/constants'; import { AppTypeEnum } from '@fastgpt/global/core/app/constants'; import { AppChatConfigType, VariableItemType } from '@fastgpt/global/core/app/type'; import { FlowNodeInputItemType } from '@fastgpt/global/core/workflow/type/io'; +import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; type ContextProps = { showRouteToAppDetail: boolean; showRouteToDatasetDetail: boolean; isShowReadRawSource: boolean; + // isShowFullText: boolean; showNodeStatus: boolean; }; type ChatBoxDataType = { @@ -30,6 +32,21 @@ type ChatBoxDataType = { }; }; +export type metadataType = { + collectionId: string; + collectionIdList: string[]; + chatItemId: string; + sourceId: string; + sourceName: string; + datasetId: string; +}; + +export type QuoteDataType = { + rawSearch: SearchDataResponseItemType[]; + metadata: metadataType; + chatTime: Date; +}; + type ChatItemContextType = { ChatBoxRef: React.RefObject | null; variablesForm: UseFormReturn; @@ -43,6 +60,9 @@ type ChatItemContextType = { chatBoxData: ChatBoxDataType; setChatBoxData: React.Dispatch>; isPlugin: boolean; + + quoteData?: QuoteDataType; + setQuoteData: React.Dispatch>; } & ContextProps; export const ChatItemContext = createContext({ @@ -61,6 +81,11 @@ export const ChatItemContext = createContext({ }, clearChatRecords: function (): void { throw new Error('Function not implemented.'); + }, + + quoteData: undefined, + setQuoteData: function (value: React.SetStateAction): void { + throw new Error('Function not implemented.'); } }); @@ -72,13 +97,14 @@ const ChatItemContextProvider = ({ showRouteToAppDetail, showRouteToDatasetDetail, isShowReadRawSource, + // isShowFullText, showNodeStatus }: { children: ReactNode; } & ContextProps) => { const ChatBoxRef = useRef(null); const variablesForm = useForm(); - + const [quoteData, setQuoteData] = useState(); const [chatBoxData, setChatBoxData] = useState({ ...defaultChatData }); @@ -131,7 +157,11 @@ const ChatItemContextProvider = ({ showRouteToAppDetail, showRouteToDatasetDetail, isShowReadRawSource, - showNodeStatus + // isShowFullText, + showNodeStatus, + + quoteData, + setQuoteData }; }, [ chatBoxData, @@ -143,7 +173,10 @@ const ChatItemContextProvider = ({ showRouteToAppDetail, showRouteToDatasetDetail, isShowReadRawSource, - showNodeStatus + // isShowFullText, + showNodeStatus, + quoteData, + setQuoteData ]); return {children}; diff --git a/projects/app/src/web/core/dataset/api.ts b/projects/app/src/web/core/dataset/api.ts index 26cd071af..92403e5dc 100644 --- a/projects/app/src/web/core/dataset/api.ts +++ b/projects/app/src/web/core/dataset/api.ts @@ -65,6 +65,7 @@ import type { listExistIdResponse } from '@/pages/api/core/dataset/apiDataset/listExistId'; import { GetQuoteDataResponse } from '@/pages/api/core/dataset/data/getQuoteData'; +import { GetQuotePermissionResponse } from '@/pages/api/core/dataset/data/getPermission'; /* ======================== dataset ======================= */ export const getDatasets = (data: GetDatasetListBody) => @@ -179,6 +180,9 @@ export const getAllTags = (datasetId: string) => export const getDatasetDataList = (data: GetDatasetDataListProps) => POST(`/core/dataset/data/v2/list`, data); +export const getDatasetDataPermission = (id?: string) => + GET(`/core/dataset/data/getPermission`, { id }); + export const getDatasetDataItemById = (id: string) => GET(`/core/dataset/data/detail`, { id });