V4.9.1 feature (#4206)
* fix: remove DefaultTeam (#4037) * fix ：Get application bound knowledge base information logical rewrite (#4057) * fix ：Get application bound knowledge base information logical rewrite * fix ：Get application bound knowledge base information logical rewrite * fix ：Get application bound knowledge base information logical rewrite * fix ：Get application bound knowledge base information logical rewrite * update package * fix: import dataset step error;perf: ai proxy avatar (#4074) * perf: pg config params * perf: ai proxy avatar * fix: import dataset step error * feat: data input ux * perf: app dataset rewite * fix: 文本提取不支持arrayString,arrayNumber等jsonSchema (#4079) * update doc ;perf: model test (#4098) * perf: extract array * update doc * perf: model test * perf: model test * perf: think tag parse (#4102) * chat quote reader (#3912) * init chat quote full text reader * linked structure * dataset data linked * optimize code * fix ts build * test finish * delete log * fix * fix ts * fix ts * remove nextId * initial scroll * fix * fix * perf: chunk read (#4109) * package * perf: chunk read * feat: api dataset support pdf parse;fix: chunk reader auth (#4117) * feat: api dataset support pdf parse * fix: chunk reader auth * feat: invitation link (#3979) * feat: invitation link schema and apis * feat: add invitation link * feat: member status: active, leave, forbidden * fix: expires show hours and minutes * feat: invalid invitation link hint * fix: typo * chore: fix typo & i18n * fix * pref: fe * feat: add ttl index for 30-day-clean-up * perf: invite member code (#4118) * perf: invite member code * fix: ts * fix: model test channel id;fix: quote reader (#4123) * fix: model test channel id * fix: quote reader * fix chat quote reader (#4125) * perf: model test;perf: sidebar trigger (#4127) * fix: import dataset step error;perf: ai proxy avatar (#4074) * perf: pg config params * perf: ai proxy avatar * fix: import dataset step error * feat: data input ux * perf: app dataset rewite * perf: model test * perf: sidebar trigger * lock * update nanoid version * fix: select component ux * fix: ts * fix: vitest * remove test * fix: prompt toolcall ui (#4139) * load log error adapt * fix: prompt toolcall ui * perf: commercial function tip * update package * pref: copy link (#4147) * fix(i18n): namespace (#4143) * hiden dataset source (#4152) * hiden dataset source * perf: reader * chore: move all tests into a single folder (#4160) * fix modal close scroll (#4162) * fix modal close scroll * update refresh * feat: rerank modal select and weight (#4164) * fix loadInitData refresh (#4169) * fix * fix * form input number default & api dataset max token * feat: mix search weight (#4170) * feat: mix search weight * feat: svg render * fix: avatar error remove (#4173) * fix: avatar error remove * fix: index * fix: guide * fix: auth * update package;fix: input data model ui (#4181) * update package * fix: ts * update config * update jieba package * add type sign * fix: input data ui * fix: page title refresh (#4186) * fix: ts * update jieba package * fix: page title refresh * fix: remove member length check when opening invite create modal (#4193) * add env to check internal ip (#4187) * fix: ts * update jieba package * add env to check internal ip * package * fix: jieba * reset package * update config * fix: jieba package * init shell * init version * change team reload * update jieba package (#4200) * update jieba package * package * update package * remove invalid code * action * package (#4201) * package * update package * remove invalid code * package * remove i18n tip (#4202) * doc (#4205) * fix: i18n (#4208) * fix: next config (#4207) * reset package * i18n * update config * i18n * remove log --------- Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com> Co-authored-by: shilin <39396378+shilin66@users.noreply.github.com> Co-authored-by: heheer <heheer@sealos.io>
This commit is contained in:
Archer
13
projects/app/src/service/core/chat/constants.ts
Normal file
13
projects/app/src/service/core/chat/constants.ts
Normal file
@@ -0,0 +1,13 @@
|
||||
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
|
||||
export const quoteDataFieldSelector = '_id q a history updateTime chunkIndex';
|
||||
|
||||
export type QuoteDataItemType = {
|
||||
_id: string;
|
||||
q: DatasetDataSchemaType['q'];
|
||||
a: DatasetDataSchemaType['a'];
|
||||
history?: DatasetDataSchemaType['history'];
|
||||
updateTime: DatasetDataSchemaType['updateTime'];
|
||||
index: DatasetDataSchemaType['chunkIndex'];
|
||||
updated?: boolean;
|
||||
};
|
||||
46
projects/app/src/service/core/chat/utils.ts
Normal file
46
projects/app/src/service/core/chat/utils.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import { QuoteDataItemType } from './constants';
|
||||
|
||||
// 获取对话时间时，引用的内容
|
||||
export function processChatTimeFilter(
|
||||
dataList: DatasetDataSchemaType[],
|
||||
chatTime: Date
|
||||
): QuoteDataItemType[] {
|
||||
return dataList.map((item) => {
|
||||
const defaultItem = {
|
||||
_id: item._id,
|
||||
q: item.q,
|
||||
a: item.a,
|
||||
updateTime: item.updateTime,
|
||||
index: item.chunkIndex
|
||||
};
|
||||
|
||||
if (!item.history) return defaultItem;
|
||||
|
||||
const history = item.history;
|
||||
const formatedChatTime = new Date(chatTime);
|
||||
|
||||
if (item.updateTime <= formatedChatTime) {
|
||||
return defaultItem;
|
||||
}
|
||||
|
||||
const latestHistoryIndex = history.findIndex(
|
||||
(historyItem) => historyItem.updateTime <= formatedChatTime
|
||||
);
|
||||
|
||||
if (latestHistoryIndex === -1) {
|
||||
return defaultItem;
|
||||
}
|
||||
|
||||
const latestHistory = history[latestHistoryIndex];
|
||||
|
||||
return {
|
||||
_id: item._id,
|
||||
q: latestHistory.q,
|
||||
a: latestHistory.a,
|
||||
updateTime: latestHistory.updateTime,
|
||||
index: item.chunkIndex,
|
||||
updated: true
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
} from '@fastgpt/global/core/dataset/controller';
|
||||
import { insertDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
|
||||
import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
|
||||
import { jiebaSplit } from '@fastgpt/service/common/string/jieba';
|
||||
import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index';
|
||||
import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
|
||||
import { DatasetDataIndexItemType, DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
|
||||
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
|
||||
@@ -138,7 +138,7 @@ export async function insertData2Dataset({
|
||||
datasetId,
|
||||
collectionId,
|
||||
dataId: _id,
|
||||
fullTextToken: jiebaSplit({ text: `${q}\n${a}`.trim() })
|
||||
fullTextToken: await jiebaSplit({ text: `${q}\n${a}`.trim() })
|
||||
}
|
||||
],
|
||||
{ session, ordered: true }
|
||||
@@ -221,10 +221,11 @@ export async function updateData2Dataset({
|
||||
}
|
||||
|
||||
// 4. Update mongo updateTime(便于脏数据检查器识别)
|
||||
const updateTime = mongoData.updateTime;
|
||||
mongoData.updateTime = new Date();
|
||||
await mongoData.save();
|
||||
|
||||
// 5. Insert vector
|
||||
// 5. insert vector
|
||||
const insertResult = await Promise.all(
|
||||
patchResult
|
||||
.filter((item) => item.type === 'create' || item.type === 'update')
|
||||
@@ -249,9 +250,20 @@ export async function updateData2Dataset({
|
||||
.filter((item) => item.type !== 'delete')
|
||||
.map((item) => item.index) as DatasetDataIndexItemType[];
|
||||
|
||||
// console.log(clonePatchResult2Insert);
|
||||
// 6. update mongo data
|
||||
await mongoSessionRun(async (session) => {
|
||||
// Update MongoData
|
||||
// Update history
|
||||
mongoData.history =
|
||||
q !== mongoData.q || a !== mongoData.a
|
||||
? [
|
||||
{
|
||||
q: mongoData.q,
|
||||
a: mongoData.a,
|
||||
updateTime: updateTime
|
||||
},
|
||||
...(mongoData.history?.slice(0, 9) || [])
|
||||
]
|
||||
: mongoData.history;
|
||||
mongoData.q = q || mongoData.q;
|
||||
mongoData.a = a ?? mongoData.a;
|
||||
mongoData.indexes = newIndexes;
|
||||
@@ -260,7 +272,7 @@ export async function updateData2Dataset({
|
||||
// update mongo data text
|
||||
await MongoDatasetDataText.updateOne(
|
||||
{ dataId: mongoData._id },
|
||||
{ fullTextToken: jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }) },
|
||||
{ fullTextToken: await jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }) },
|
||||
{ session }
|
||||
);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ChatSchema } from '@fastgpt/global/core/chat/type';
|
||||
import { AIChatItemType, ChatHistoryItemResType, ChatSchema } from '@fastgpt/global/core/chat/type';
|
||||
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
|
||||
import { AuthModeType } from '@fastgpt/service/support/permission/type';
|
||||
import { authOutLink } from './outLink';
|
||||
@@ -6,6 +6,8 @@ import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
|
||||
import { authTeamSpaceToken } from './team';
|
||||
import { AuthUserTypeEnum, ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { authApp } from '@fastgpt/service/support/permission/app/auth';
|
||||
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
|
||||
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
|
||||
|
||||
/*
|
||||
检查chat的权限：
|
||||
@@ -23,6 +25,14 @@ const defaultResponseShow = {
|
||||
showNodeStatus: true,
|
||||
showRawSource: true
|
||||
};
|
||||
type AuthChatCommonProps = {
|
||||
appId: string;
|
||||
shareId?: string;
|
||||
outLinkUid?: string;
|
||||
teamId?: string;
|
||||
teamToken?: string;
|
||||
};
|
||||
|
||||
export async function authChatCrud({
|
||||
appId,
|
||||
chatId,
|
||||
@@ -33,14 +43,10 @@ export async function authChatCrud({
|
||||
teamId: spaceTeamId,
|
||||
teamToken,
|
||||
...props
|
||||
}: AuthModeType & {
|
||||
appId: string;
|
||||
chatId?: string;
|
||||
shareId?: string;
|
||||
outLinkUid?: string;
|
||||
teamId?: string;
|
||||
teamToken?: string;
|
||||
}): Promise<{
|
||||
}: AuthModeType &
|
||||
AuthChatCommonProps & {
|
||||
chatId?: string;
|
||||
}): Promise<{
|
||||
teamId: string;
|
||||
tmbId: string;
|
||||
uid: string;
|
||||
@@ -184,3 +190,57 @@ export async function authChatCrud({
|
||||
|
||||
return Promise.reject(ChatErrEnum.unAuthChat);
|
||||
}
|
||||
|
||||
export const authCollectionInChat = async ({
|
||||
collectionIds,
|
||||
appId,
|
||||
chatId,
|
||||
chatItemDataId
|
||||
}: {
|
||||
collectionIds: string[];
|
||||
appId: string;
|
||||
chatId: string;
|
||||
chatItemDataId: string;
|
||||
}): Promise<{
|
||||
chatItem: { time: Date; responseData?: ChatHistoryItemResType[] };
|
||||
}> => {
|
||||
try {
|
||||
const chatItem = (await MongoChatItem.findOne(
|
||||
{
|
||||
appId,
|
||||
chatId,
|
||||
dataId: chatItemDataId
|
||||
},
|
||||
'responseData time'
|
||||
).lean()) as { time: Date; responseData?: ChatHistoryItemResType[] };
|
||||
|
||||
if (!chatItem) return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
|
||||
|
||||
// 找 responseData 里，是否有该文档 id
|
||||
const responseData = chatItem.responseData || [];
|
||||
const flatResData: ChatHistoryItemResType[] =
|
||||
responseData
|
||||
?.map((item) => {
|
||||
return [
|
||||
item,
|
||||
...(item.pluginDetail || []),
|
||||
...(item.toolDetail || []),
|
||||
...(item.loopDetail || [])
|
||||
];
|
||||
})
|
||||
.flat() || [];
|
||||
|
||||
const quoteListSet = new Set(
|
||||
flatResData
|
||||
.map((item) => item.quoteList?.map((quote) => String(quote.collectionId)) || [])
|
||||
.flat()
|
||||
);
|
||||
|
||||
if (collectionIds.every((id) => quoteListSet.has(id))) {
|
||||
return {
|
||||
chatItem
|
||||
};
|
||||
}
|
||||
} catch (error) {}
|
||||
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user