gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: chunk read (#4109)

Browse Source 
* package

* perf: chunk read
This commit is contained in:
Archer
2025-03-11 21:58:24 +08:00
committed by archer
parent ac7091f8d6
commit 30f83f848d
29 changed files with 416 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
packages/service/core/chat/saveChat.ts
View File

@@ -74,41 +74,42 @@ export async function saveChat({
(node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput
)?.inputs;
await mongoSessionRun(async (session) => {
const processedContent = content.map((item) => {
if (item.obj === ChatRoleEnum.AI) {
const nodeResponse = item[DispatchNodeResponseKeyEnum.nodeResponse];
// Format save chat content: Remove quote q/a
const processedContent = content.map((item) => {
if (item.obj === ChatRoleEnum.AI) {
const nodeResponse = item[DispatchNodeResponseKeyEnum.nodeResponse];
if (nodeResponse) {
return {
...item,
[DispatchNodeResponseKeyEnum.nodeResponse]: nodeResponse.map((responseItem) => {
if (
responseItem.moduleType === FlowNodeTypeEnum.datasetSearchNode &&
responseItem.quoteList
) {
return {
...item,
quoteList: responseItem.quoteList.map((quote: any) => ({
id: quote.id,
chunkIndex: quote.chunkIndex,
datasetId: quote.datasetId,
collectionId: quote.collectionId,
sourceId: quote.sourceId,
sourceName: quote.sourceName,
score: quote.score,
tokens: quote.tokens
}))
};
}
return item;
})
};
}
if (nodeResponse) {
return {
...item,
[DispatchNodeResponseKeyEnum.nodeResponse]: nodeResponse.map((responseItem) => {
if (
responseItem.moduleType === FlowNodeTypeEnum.datasetSearchNode &&
responseItem.quoteList
) {
return {
...responseItem,
quoteList: responseItem.quoteList.map((quote: any) => ({
id: quote.id,
chunkIndex: quote.chunkIndex,
datasetId: quote.datasetId,
collectionId: quote.collectionId,
sourceId: quote.sourceId,
sourceName: quote.sourceName,
score: quote.score,
tokens: quote.tokens
}))
};
}
return responseItem;
})
};
}
return item;
});
}
return item;
});
await mongoSessionRun(async (session) => {
const [{ _id: chatItemIdHuman }, { _id: chatItemIdAi }] = await MongoChatItem.insertMany(
processedContent.map((item) => ({
chatId,

4
packages/service/core/dataset/data/schema.ts
View File

@@ -98,15 +98,11 @@ try {
chunkIndex: 1,
updateTime: -1
});
// FullText tmp full text index
// DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
// Recall vectors after data matching
DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
DatasetDataSchema.index({ updateTime: 1 });
// rebuild data
DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
DatasetDataSchema.index({ initFullText: 1 });
} catch (error) {
console.log(error);
}