feat: api dataset support pdf parse;fix: chunk reader auth (#4117)

* feat: api dataset support pdf parse * fix: chunk reader auth
2025-03-12 12:41:19 +08:00
parent 30f83f848d
commit 1a3613cd2c
27 changed files with 378 additions and 355 deletions
--- a/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx
+++ b/projects/app/src/pageComponents/app/detail/Logs/DetailLogsModal.tsx
@@ -184,7 +184,6 @@ const DetailLogsModal = ({ appId, chatId, onClose }: Props) => {
              borderRadius={'md'}
            >
              <ChatQuoteList
-                chatTime={quoteData.chatTime}
                rawSearch={quoteData.rawSearch}
                metadata={quoteData.metadata}
                onClose={() => setQuoteData(undefined)}
--- a/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx
+++ b/projects/app/src/pageComponents/app/detail/SimpleApp/ChatTest.tsx
@@ -88,7 +88,6 @@ const ChatTest = ({ appForm, setRenderEdit }: Props) => {
      {quoteData && (
        <Box flex={'1 0 0'} w={0} maxW={'560px'} {...cardStyles} boxShadow={'3'}>
          <ChatQuoteList
-            chatTime={quoteData.chatTime}
            rawSearch={quoteData.rawSearch}
            metadata={quoteData.metadata}
            onClose={() => setQuoteData(undefined)}
--- a/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/ChatTest.tsx
+++ b/projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/ChatTest.tsx
@@ -163,7 +163,6 @@ const ChatTest = ({ isOpen, nodes = [], edges = [], onClose }: Props) => {
              borderRadius={'md'}
            >
              <ChatQuoteList
-                chatTime={quoteData.chatTime}
                rawSearch={quoteData.rawSearch}
                metadata={quoteData.metadata}
                onClose={() => setQuoteData(undefined)}
--- a/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx
+++ b/projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx
@@ -1,4 +1,4 @@
-import { Box, Button, Flex, HStack } from '@chakra-ui/react';
+import { Box, Flex, HStack } from '@chakra-ui/react';
 import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
 import MyIcon from '@fastgpt/web/components/common/Icon';
@@ -8,41 +8,34 @@ import DownloadButton from './DownloadButton';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { downloadFetch } from '@/web/common/system/utils';
 import { useCallback, useEffect, useMemo, useState } from 'react';
-import { getCollectionSource, getDatasetDataPermission } from '@/web/core/dataset/api';
-import { useChatStore } from '@/web/core/chat/context/useChatStore';
-import { useToast } from '@fastgpt/web/hooks/useToast';
-import { getErrText } from '@fastgpt/global/common/error/utils';
+import { getDatasetDataPermission } from '@/web/core/dataset/api';
 import ScoreTag from './ScoreTag';
 import { formatScore } from '@/components/core/dataset/QuoteItem';
 import NavButton from './NavButton';
 import { useLinkedScroll } from '@fastgpt/web/hooks/useLinkedScroll';
 import CollectionQuoteItem from './CollectionQuoteItem';
-import { DatasetDataListItemType } from '@/global/core/dataset/type';
-import { metadataType } from '@/web/core/chat/context/chatItemContext';
+import { GetCollectionQuoteDataProps } from '@/web/core/chat/context/chatItemContext';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import { getCollectionQuote } from '@/web/core/chat/api';
 import MyIconButton from '@fastgpt/web/components/common/Icon/button';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 import MyBox from '@fastgpt/web/components/common/MyBox';
 import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource';
+import { QuoteDataItemType } from '@/service/core/chat/constants';

 const CollectionReader = ({
  rawSearch,
  metadata,
-  chatTime,
  onClose
 }: {
  rawSearch: SearchDataResponseItemType[];
-  metadata: metadataType;
-  chatTime: Date;
+  metadata: GetCollectionQuoteDataProps;
  onClose: () => void;
 }) => {
  const { t } = useTranslation();
  const router = useRouter();
-  const { toast } = useToast();
-  const { chatId, appId, outLinkAuthData } = useChatStore();
  const { userInfo } = useUserStore();
-  const { collectionId, datasetId, chatItemId, sourceId, sourceName } = metadata;
+  const { collectionId, datasetId, chatItemDataId, sourceId, sourceName } = metadata;
  const [quoteIndex, setQuoteIndex] = useState(0);

  const { data: permissionData, loading: isPermissionLoading } = useRequest2(
@@ -75,11 +68,10 @@ const CollectionReader = ({
    refreshDeps: [collectionId],
    params: {
      collectionId,
-      chatTime,
-      chatItemId,
-      chatId,
-      appId,
-      ...outLinkAuthData
+      chatItemDataId,
+      chatId: metadata.chatId,
+      appId: metadata.appId,
+      ...metadata.outLinkAuthData
    },
    initialId: currentQuoteItem?.id,
    initialIndex: currentQuoteItem?.chunkIndex,
@@ -87,11 +79,14 @@ const CollectionReader = ({
  });

  const loading = isLoading || isPermissionLoading;
-  const isDeleted = !datasetDataList.find((item) => item._id === currentQuoteItem?.id);
+  const isDeleted = useMemo(
+    () => !datasetDataList.find((item) => item._id === currentQuoteItem?.id),
+    [datasetDataList, currentQuoteItem?.id]
+  );

  const formatedDataList = useMemo(
    () =>
-      datasetDataList.map((item: DatasetDataListItemType) => {
+      datasetDataList.map((item: QuoteDataItemType) => {
        const isCurrentSelected = currentQuoteItem?.id === item._id;
        const quoteIndex = filterResults.findIndex((res) => res.id === item._id);

@@ -115,17 +110,12 @@ const CollectionReader = ({
      filename: 'data.csv',
      body: {
        collectionId: collectionId,
-        chatTime: chatTime,
-        chatItemId: chatItemId
+        chatItemDataId
      }
    });
  });

-  const handleRead = getCollectionSourceAndOpen({
-    appId,
-    chatId,
-    ...metadata
-  });
+  const handleRead = getCollectionSourceAndOpen(metadata);

  const handleNavigate = useCallback(
    async (targetIndex: number) => {
--- a/projects/app/src/pageComponents/chat/ChatQuoteList/QuoteReader.tsx
+++ b/projects/app/src/pageComponents/chat/ChatQuoteList/QuoteReader.tsx
@@ -4,75 +4,60 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
 import MyBox from '@fastgpt/web/components/common/MyBox';
 import { useTranslation } from 'react-i18next';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
-import { useChatStore } from '@/web/core/chat/context/useChatStore';
 import QuoteItem from './QuoteItem';
 import { useMemo } from 'react';
 import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
 import { formatScore } from '@/components/core/dataset/QuoteItem';
-import { metadataType } from '@/web/core/chat/context/chatItemContext';
+import { GetAllQuoteDataProps } from '@/web/core/chat/context/chatItemContext';
 import { getQuoteDataList } from '@/web/core/chat/api';

 const QuoteReader = ({
  rawSearch,
  metadata,
-  chatTime,
  onClose
 }: {
  rawSearch: SearchDataResponseItemType[];
-  metadata: metadataType;
-  chatTime: Date;
+  metadata: GetAllQuoteDataProps;
  onClose: () => void;
 }) => {
  const { t } = useTranslation();

-  const { chatId, appId, outLinkAuthData } = useChatStore();
-
-  const { data, loading } = useRequest2(
+  const { data: quoteList, loading } = useRequest2(
    async () =>
      await getQuoteDataList({
        datasetDataIdList: rawSearch.map((item) => item.id),
-        chatTime,
        collectionIdList: metadata.collectionIdList,
-        chatItemId: metadata.chatItemId,
-        appId,
-        chatId,
-        ...outLinkAuthData
+        chatItemDataId: metadata.chatItemDataId,
+        appId: metadata.appId,
+        chatId: metadata.chatId,
+        ...metadata.outLinkAuthData
      }),
    {
      manual: false
    }
  );

-  const filterResults = useMemo(() => {
-    if (!metadata.collectionId) {
-      return rawSearch;
-    }
-
-    return rawSearch.filter(
-      (item) => item.collectionId === metadata.collectionId && item.sourceId === metadata.sourceId
-    );
-  }, [metadata, rawSearch]);
-
  const formatedDataList = useMemo(() => {
-    return filterResults
-      .map((item) => {
-        const currentFilterItem = data?.quoteList.find((res) => res._id === item.id);
+    return rawSearch
+      .map((searchItem) => {
+        const dataItem = quoteList?.find((item) => item._id === searchItem.id);

        return {
-          ...item,
-          q: currentFilterItem?.q || '',
-          a: currentFilterItem?.a || '',
-          score: formatScore(item.score),
+          id: searchItem.id,
+          q: dataItem?.q || 'Can not find Data',
+          a: dataItem?.a || '',
+          score: formatScore(searchItem.score),
+          sourceName: searchItem?.sourceName || '',
          icon: getSourceNameIcon({
-            sourceId: item.sourceId,
-            sourceName: item.sourceName
+            sourceId: searchItem.sourceId,
+            sourceName: searchItem.sourceName
          })
        };
      })
      .sort((a, b) => {
        return (b.score.primaryScore?.value || 0) - (a.score.primaryScore?.value || 0);
      });
-  }, [data?.quoteList, filterResults]);
+  }, [quoteList, rawSearch]);

  return (
    <Flex flexDirection={'column'} h={'full'}>
@@ -86,18 +71,7 @@ const QuoteReader = ({
      >
        <Box flex={1} py={4}>
          <Flex gap={2} mr={2} mb={1}>
-            <MyIcon
-              name={
-                metadata.sourceId && metadata.sourceName
-                  ? (getSourceNameIcon({
-                      sourceId: metadata.sourceId,
-                      sourceName: metadata.sourceName
-                    }) as any)
-                  : 'core/chat/quoteFill'
-              }
-              w={['1rem', '1.25rem']}
-              color={'primary.600'}
-            />
+            <MyIcon name={'core/chat/quoteFill'} w={['1rem', '1.25rem']} color={'primary.600'} />
            <Box
              maxW={['200px', '300px']}
              className={'textEllipsis'}
@@ -105,9 +79,7 @@ const QuoteReader = ({
              color={'myGray.900'}
              fontWeight={'medium'}
            >
-              {metadata.sourceName
-                ? metadata.sourceName
-                : t('common:core.chat.Quote Amount', { amount: rawSearch.length })}
+              {t('common:core.chat.Quote Amount', { amount: rawSearch.length })}
            </Box>
          </Flex>
          <Box fontSize={'mini'} color={'myGray.500'}>
--- a/projects/app/src/pageComponents/chat/ChatQuoteList/index.tsx
+++ b/projects/app/src/pageComponents/chat/ChatQuoteList/index.tsx
@@ -1,39 +1,28 @@
 import React from 'react';
 import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import { useContextSelector } from 'use-context-selector';
-import { ChatItemContext, metadataType } from '@/web/core/chat/context/chatItemContext';
+import { ChatItemContext, GetQuoteProps } from '@/web/core/chat/context/chatItemContext';
 import CollectionQuoteReader from './CollectionQuoteReader';
 import QuoteReader from './QuoteReader';

 const ChatQuoteList = ({
-  chatTime,
  rawSearch = [],
  metadata,
  onClose
 }: {
-  chatTime: Date;
  rawSearch: SearchDataResponseItemType[];
-  metadata: metadataType;
+  metadata: GetQuoteProps;
  onClose: () => void;
 }) => {
  const isShowReadRawSource = useContextSelector(ChatItemContext, (v) => v.isShowReadRawSource);

  return (
    <>
-      {metadata.collectionId && isShowReadRawSource ? (
-        <CollectionQuoteReader
-          rawSearch={rawSearch}
-          metadata={metadata}
-          chatTime={chatTime}
-          onClose={onClose}
-        />
-      ) : (
-        <QuoteReader
-          rawSearch={rawSearch}
-          metadata={metadata}
-          chatTime={chatTime}
-          onClose={onClose}
-        />
+      {'collectionId' in metadata && isShowReadRawSource && (
+        <CollectionQuoteReader rawSearch={rawSearch} metadata={metadata} onClose={onClose} />
+      )}
+      {'collectionIdList' in metadata && (
+        <QuoteReader rawSearch={rawSearch} metadata={metadata} onClose={onClose} />
      )}
    </>
  );