V4.9.1 feature (#4206)
* fix: remove DefaultTeam (#4037) * fix ：Get application bound knowledge base information logical rewrite (#4057) * fix ：Get application bound knowledge base information logical rewrite * fix ：Get application bound knowledge base information logical rewrite * fix ：Get application bound knowledge base information logical rewrite * fix ：Get application bound knowledge base information logical rewrite * update package * fix: import dataset step error;perf: ai proxy avatar (#4074) * perf: pg config params * perf: ai proxy avatar * fix: import dataset step error * feat: data input ux * perf: app dataset rewite * fix: 文本提取不支持arrayString,arrayNumber等jsonSchema (#4079) * update doc ;perf: model test (#4098) * perf: extract array * update doc * perf: model test * perf: model test * perf: think tag parse (#4102) * chat quote reader (#3912) * init chat quote full text reader * linked structure * dataset data linked * optimize code * fix ts build * test finish * delete log * fix * fix ts * fix ts * remove nextId * initial scroll * fix * fix * perf: chunk read (#4109) * package * perf: chunk read * feat: api dataset support pdf parse;fix: chunk reader auth (#4117) * feat: api dataset support pdf parse * fix: chunk reader auth * feat: invitation link (#3979) * feat: invitation link schema and apis * feat: add invitation link * feat: member status: active, leave, forbidden * fix: expires show hours and minutes * feat: invalid invitation link hint * fix: typo * chore: fix typo & i18n * fix * pref: fe * feat: add ttl index for 30-day-clean-up * perf: invite member code (#4118) * perf: invite member code * fix: ts * fix: model test channel id;fix: quote reader (#4123) * fix: model test channel id * fix: quote reader * fix chat quote reader (#4125) * perf: model test;perf: sidebar trigger (#4127) * fix: import dataset step error;perf: ai proxy avatar (#4074) * perf: pg config params * perf: ai proxy avatar * fix: import dataset step error * feat: data input ux * perf: app dataset rewite * perf: model test * perf: sidebar trigger * lock * update nanoid version * fix: select component ux * fix: ts * fix: vitest * remove test * fix: prompt toolcall ui (#4139) * load log error adapt * fix: prompt toolcall ui * perf: commercial function tip * update package * pref: copy link (#4147) * fix(i18n): namespace (#4143) * hiden dataset source (#4152) * hiden dataset source * perf: reader * chore: move all tests into a single folder (#4160) * fix modal close scroll (#4162) * fix modal close scroll * update refresh * feat: rerank modal select and weight (#4164) * fix loadInitData refresh (#4169) * fix * fix * form input number default & api dataset max token * feat: mix search weight (#4170) * feat: mix search weight * feat: svg render * fix: avatar error remove (#4173) * fix: avatar error remove * fix: index * fix: guide * fix: auth * update package;fix: input data model ui (#4181) * update package * fix: ts * update config * update jieba package * add type sign * fix: input data ui * fix: page title refresh (#4186) * fix: ts * update jieba package * fix: page title refresh * fix: remove member length check when opening invite create modal (#4193) * add env to check internal ip (#4187) * fix: ts * update jieba package * add env to check internal ip * package * fix: jieba * reset package * update config * fix: jieba package * init shell * init version * change team reload * update jieba package (#4200) * update jieba package * package * update package * remove invalid code * action * package (#4201) * package * update package * remove invalid code * package * remove i18n tip (#4202) * doc (#4205) * fix: i18n (#4208) * fix: next config (#4207) * reset package * i18n * update config * i18n * remove log --------- Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com> Co-authored-by: shilin <39396378+shilin66@users.noreply.github.com> Co-authored-by: heheer <heheer@sealos.io>
This commit is contained in:
Archer
@@ -6,7 +6,7 @@ import {
|
||||
} from '@fastgpt/global/core/dataset/controller';
|
||||
import { insertDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
|
||||
import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
|
||||
import { jiebaSplit } from '@fastgpt/service/common/string/jieba';
|
||||
import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index';
|
||||
import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
|
||||
import { DatasetDataIndexItemType, DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
|
||||
import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
|
||||
@@ -138,7 +138,7 @@ export async function insertData2Dataset({
|
||||
datasetId,
|
||||
collectionId,
|
||||
dataId: _id,
|
||||
fullTextToken: jiebaSplit({ text: `${q}\n${a}`.trim() })
|
||||
fullTextToken: await jiebaSplit({ text: `${q}\n${a}`.trim() })
|
||||
}
|
||||
],
|
||||
{ session, ordered: true }
|
||||
@@ -221,10 +221,11 @@ export async function updateData2Dataset({
|
||||
}
|
||||
|
||||
// 4. Update mongo updateTime(便于脏数据检查器识别)
|
||||
const updateTime = mongoData.updateTime;
|
||||
mongoData.updateTime = new Date();
|
||||
await mongoData.save();
|
||||
|
||||
// 5. Insert vector
|
||||
// 5. insert vector
|
||||
const insertResult = await Promise.all(
|
||||
patchResult
|
||||
.filter((item) => item.type === 'create' || item.type === 'update')
|
||||
@@ -249,9 +250,20 @@ export async function updateData2Dataset({
|
||||
.filter((item) => item.type !== 'delete')
|
||||
.map((item) => item.index) as DatasetDataIndexItemType[];
|
||||
|
||||
// console.log(clonePatchResult2Insert);
|
||||
// 6. update mongo data
|
||||
await mongoSessionRun(async (session) => {
|
||||
// Update MongoData
|
||||
// Update history
|
||||
mongoData.history =
|
||||
q !== mongoData.q || a !== mongoData.a
|
||||
? [
|
||||
{
|
||||
q: mongoData.q,
|
||||
a: mongoData.a,
|
||||
updateTime: updateTime
|
||||
},
|
||||
...(mongoData.history?.slice(0, 9) || [])
|
||||
]
|
||||
: mongoData.history;
|
||||
mongoData.q = q || mongoData.q;
|
||||
mongoData.a = a ?? mongoData.a;
|
||||
mongoData.indexes = newIndexes;
|
||||
@@ -260,7 +272,7 @@ export async function updateData2Dataset({
|
||||
// update mongo data text
|
||||
await MongoDatasetDataText.updateOne(
|
||||
{ dataId: mongoData._id },
|
||||
{ fullTextToken: jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }) },
|
||||
{ fullTextToken: await jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }) },
|
||||
{ session }
|
||||
);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user