perf: chunk read (#4109)

* package

* perf: chunk read
This commit is contained in:
Archer
2025-03-11 21:58:24 +08:00
committed by archer
parent ac7091f8d6
commit 30f83f848d
29 changed files with 416 additions and 431 deletions

View File

@@ -6,6 +6,8 @@ import { ApiRequestProps } from '@fastgpt/service/type/next';
import { LinkedListResponse, LinkedPaginationProps } from '@fastgpt/web/common/fetch/type';
import { FilterQuery, Types } from 'mongoose';
import { dataFieldSelector, processChatTimeFilter } from './getQuote';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
export type GetCollectionQuoteProps = LinkedPaginationProps & {
chatTime: Date;
@@ -36,8 +38,8 @@ async function handler(
nextId,
nextIndex,
chatTime,
isInitialLoad,
collectionId,
chatItemId,
appId,
@@ -48,21 +50,32 @@ async function handler(
teamToken,
pageSize = 15
} = req.body;
const limitedPageSize = Math.min(pageSize, 30);
await Promise.all([
authChatCrud({
try {
await authDatasetCollection({
req,
authToken: true,
appId,
chatId,
shareId,
outLinkUid,
teamId,
teamToken
}),
authCollectionInChat({ appId, chatId, chatItemId, collectionId })
]);
authApiKey: true,
collectionId: req.body.collectionId,
per: ReadPermissionVal
});
} catch (error) {
await Promise.all([
authChatCrud({
req,
authToken: true,
appId,
chatId,
shareId,
outLinkUid,
teamId,
teamToken
}),
authCollectionInChat({ appId, chatId, chatItemId, collectionId })
]);
}
const baseMatch: BaseMatchType = {
collectionId,

View File

@@ -71,9 +71,7 @@ async function handler(req: ApiRequestProps<GetQuoteDataProps>): Promise<GetQuot
export default NextAPI(handler);
export function processChatTimeFilter(list: DatasetDataSchemaType[], chatTime?: Date) {
if (!chatTime) return list;
export function processChatTimeFilter(list: DatasetDataSchemaType[], chatTime: Date) {
return list.map((item) => {
if (!item.history) return item;

View File

@@ -39,7 +39,7 @@ async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: Next
};
res.setHeader('Content-Type', 'text/csv; charset=utf-8;');
res.setHeader('Content-Disposition', 'attachment; filename=usage.csv; ');
res.setHeader('Content-Disposition', 'attachment; filename=data.csv; ');
const cursor = MongoDatasetData.find(where, 'q a', {
...readFromSecondary,
@@ -54,10 +54,13 @@ async function handler(req: ApiRequestProps<ExportCollectionBody, {}>, res: Next
readStream: cursor
});
cursor.on('data', (doc) => {
const res = doc.a ? `\n${doc.q}\n${doc.a}` : `\n${doc.q}`;
write(`\uFEFFindex,content`);
write(res);
cursor.on('data', (doc) => {
const q = doc.q.replace(/"/g, '""') || '';
const a = doc.a.replace(/"/g, '""') || '';
write(`\n"${q}","${a}"`);
});
cursor.on('end', () => {

View File

@@ -168,66 +168,68 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
<NextHead title={chatBoxData.app.name} icon={chatBoxData.app.avatar}></NextHead>
{/* pc show myself apps */}
{isPc && (
<Box borderRight={theme.borders.base} w={'220px'} flexShrink={0}>
<Box borderRight={theme.borders.base} flex={'0 0 220px'}>
<SliderApps apps={myApps} activeAppId={appId} />
</Box>
)}
<PageContainer
isLoading={loading}
flex={'1 0 0'}
w={0}
p={[0, '16px']}
pr={quoteData ? '8px !important' : '16px'}
position={'relative'}
>
<Flex h={'100%'} flexDirection={['column', 'row']}>
{/* pc always show history. */}
{RenderHistorySlider}
{/* chat container */}
<Flex
position={'relative'}
h={[0, '100%']}
w={['100%', 0]}
flex={'1 0 0'}
flexDirection={'column'}
>
{/* header */}
<ChatHeader
totalRecordsCount={totalRecordsCount}
apps={myApps}
history={chatRecords}
showHistory
/>
{(!quoteData || isPc) && (
<PageContainer
isLoading={loading}
flex={'1 0 0'}
w={0}
p={[0, '16px']}
position={'relative'}
>
<Flex h={'100%'} flexDirection={['column', 'row']}>
{/* pc always show history. */}
{RenderHistorySlider}
{/* chat container */}
<Flex
position={'relative'}
h={[0, '100%']}
w={['100%', 0]}
flex={'1 0 0'}
flexDirection={'column'}
>
{/* header */}
<ChatHeader
totalRecordsCount={totalRecordsCount}
apps={myApps}
history={chatRecords}
showHistory
/>
{/* chat box */}
<Box flex={'1 0 0'} bg={'white'}>
{isPlugin ? (
<CustomPluginRunBox
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
onNewChat={() => onChangeChatId(getNanoid())}
onStartChat={onStartChat}
/>
) : (
<ChatBox
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
showEmptyIntro
feedbackType={'user'}
onStartChat={onStartChat}
chatType={'chat'}
isReady={!loading}
/>
)}
</Box>
{/* chat box */}
<Box flex={'1 0 0'} bg={'white'}>
{isPlugin ? (
<CustomPluginRunBox
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
onNewChat={() => onChangeChatId(getNanoid())}
onStartChat={onStartChat}
/>
) : (
<ChatBox
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
showEmptyIntro
feedbackType={'user'}
onStartChat={onStartChat}
chatType={'chat'}
isReady={!loading}
/>
)}
</Box>
</Flex>
</Flex>
</Flex>
</PageContainer>
</PageContainer>
)}
{quoteData && (
<PageContainer w={['full', '588px']} insertProps={{ bg: 'white' }}>
<PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}

View File

@@ -239,73 +239,80 @@ const OutLink = (props: Props) => {
}, [isOpenSlider, isPc, onCloseSlider, quoteData, showHistory, t]);
return (
<Box h={'full'} display={quoteData ? 'flex' : ''}>
<>
<NextHead
title={props.appName || data?.app?.name || 'AI'}
desc={props.appIntro || data?.app?.intro}
icon={props.appAvatar || data?.app?.avatar}
/>
<PageContainer
isLoading={loading}
{...(isEmbed
? { p: '0 !important', insertProps: { borderRadius: '0', boxShadow: 'none' } }
: { p: [0, 5] })}
>
<Flex h={'100%'} flexDirection={['column', 'row']}>
{RenderHistoryList}
{/* chat container */}
<Flex
position={'relative'}
h={[0, '100%']}
w={['100%', 0]}
<Flex h={'full'}>
{(!quoteData || isPc) && (
<PageContainer
flex={'1 0 0'}
flexDirection={'column'}
w={0}
isLoading={loading}
{...(isEmbed
? { p: '0 !important', insertProps: { borderRadius: '0', boxShadow: 'none' } }
: { p: [0, 5] })}
>
{/* header */}
{showHead === '1' ? (
<ChatHeader
history={chatRecords}
totalRecordsCount={totalRecordsCount}
showHistory={showHistory === '1'}
/>
) : null}
{/* chat box */}
<Box flex={1} bg={'white'}>
{isPlugin ? (
<CustomPluginRunBox
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
onNewChat={() => onChangeChatId(getNanoid())}
onStartChat={startChat}
/>
) : (
<ChatBox
isReady={!loading}
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
feedbackType={'user'}
onStartChat={startChat}
chatType="share"
/>
)}
</Box>
</Flex>
</Flex>
</PageContainer>
{quoteData && (
<PageContainer w={['full', '800px']} py={5}>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}
metadata={quoteData.metadata}
onClose={() => setQuoteData(undefined)}
/>
</PageContainer>
)}
</Box>
<Flex h={'100%'} flexDirection={['column', 'row']}>
{RenderHistoryList}
{/* chat container */}
<Flex
position={'relative'}
h={[0, '100%']}
w={['100%', 0]}
flex={'1 0 0'}
flexDirection={'column'}
>
{/* header */}
{showHead === '1' ? (
<ChatHeader
history={chatRecords}
totalRecordsCount={totalRecordsCount}
showHistory={showHistory === '1'}
/>
) : null}
{/* chat box */}
<Box flex={1} bg={'white'}>
{isPlugin ? (
<CustomPluginRunBox
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
onNewChat={() => onChangeChatId(getNanoid())}
onStartChat={startChat}
/>
) : (
<ChatBox
isReady={!loading}
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
feedbackType={'user'}
onStartChat={startChat}
chatType="share"
/>
)}
</Box>
</Flex>
</Flex>
</PageContainer>
)}
{quoteData && (
<PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}
metadata={quoteData.metadata}
onClose={() => setQuoteData(undefined)}
/>
</PageContainer>
)}
</Flex>
</>
);
};

View File

@@ -191,52 +191,59 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
</Box>
)}
<PageContainer isLoading={loading} flex={'1 0 0'} w={0} p={[0, '16px']} position={'relative'}>
<Flex h={'100%'} flexDirection={['column', 'row']} bg={'white'}>
{RenderHistoryList}
{/* chat container */}
<Flex
position={'relative'}
h={[0, '100%']}
w={['100%', 0]}
flex={'1 0 0'}
flexDirection={'column'}
>
{/* header */}
<ChatHeader
totalRecordsCount={totalRecordsCount}
apps={myApps}
history={chatRecords}
showHistory
/>
{/* chat box */}
<Box flex={1}>
{chatBoxData.app.type === AppTypeEnum.plugin ? (
<CustomPluginRunBox
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
onNewChat={() => onChangeChatId(getNanoid())}
onStartChat={startChat}
/>
) : (
<ChatBox
isReady={!loading}
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
feedbackType={'user'}
onStartChat={startChat}
chatType="team"
/>
)}
</Box>
{(!quoteData || isPc) && (
<PageContainer
isLoading={loading}
flex={'1 0 0'}
w={0}
p={[0, '16px']}
position={'relative'}
>
<Flex h={'100%'} flexDirection={['column', 'row']} bg={'white'}>
{RenderHistoryList}
{/* chat container */}
<Flex
position={'relative'}
h={[0, '100%']}
w={['100%', 0]}
flex={'1 0 0'}
flexDirection={'column'}
>
{/* header */}
<ChatHeader
totalRecordsCount={totalRecordsCount}
apps={myApps}
history={chatRecords}
showHistory
/>
{/* chat box */}
<Box flex={1}>
{chatBoxData.app.type === AppTypeEnum.plugin ? (
<CustomPluginRunBox
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
onNewChat={() => onChangeChatId(getNanoid())}
onStartChat={startChat}
/>
) : (
<ChatBox
isReady={!loading}
appId={appId}
chatId={chatId}
outLinkAuthData={outLinkAuthData}
feedbackType={'user'}
onStartChat={startChat}
chatType="team"
/>
)}
</Box>
</Flex>
</Flex>
</Flex>
</PageContainer>
</PageContainer>
)}
{quoteData && (
<PageContainer w={['full', '800px']} py={5}>
<PageContainer flex={'1 0 0'} w={0} maxW={'560px'}>
<ChatQuoteList
chatTime={quoteData.chatTime}
rawSearch={quoteData.rawSearch}