V4.8.18 feature (#3565)
* feat: org CRUD (#3380) * feat: add org schema * feat: org manage UI * feat: OrgInfoModal * feat: org tree view * feat: org management * fix: init root org * feat: org permission for app * feat: org support for dataset * fix: disable org role control * styles: opt type signatures * fix: remove unused permission * feat: delete org collaborator * perf: Team org ui (#3499) * perf: org ui * perf: org ui * feat: org auth for app & dataset (#3498) * feat: auth org resource permission * feat: org auth support for app & dataset * perf: org permission check (#3500) * i18n (#3501) * name * i18n * feat: support dataset changeOwner (#3483) * feat: support dataset changeOwner * chore: update dataset change owner api * feat: permission manage UI for org (#3503) * perf: password check;perf: image upload check;perf: sso login check (#3509) * perf: password check * perf: image upload check * perf: sso login check * force show update notification modal & fix login page text (#3512) * fix login page English text * update notification modal * perf: notify account (#3515) * perf(plugin): improve searXNG empty result handling and documentation (#3507) * perf(plugin): improve searXNG empty result handling and documentation * 修改了文档和代码部分无搜索的结果的反馈 * refactor: org pathId (#3516) * optimize payment process (#3517) * feat: support wecom sso (#3518) * feat: support wecom sso * chore: remove unused wecom js-sdk dependency * fix qrcode script (#3520) * fix qrcode script * i18n * perf: full text collection and search code;perf: rename function (#3519) * perf: full text collection and search code * perf: rename function * perf: notify modal * remove invalid code * perf: sso login * perf: pay process * 4.8.18 test (#3524) * perf: remove local token * perf: index * perf: file encoding;perf: leave team code;@c121914yu perf: full text search code (#3528) * perf: text encoding * perf: leave team code * perf: full text search code * fix: http status * perf: embedding search and vector avatar * perf: async read file (#3531) * refactor: team permission manager (#3535) * perf: classify org, group and member * refactor: team per manager * fix: missing functions * 4.8.18 test (#3543) * perf: login check * doc * perf: llm model config * perf: team clb config * fix: MemberModal UI (#3553) * fix: adapt MemberModal title and icon * fix: adapt member modal * fix: search input placeholder * fix: add button text * perf: org permission (#3556) * docs:用户答疑的官方文档补充 (#3540) * docs:用户答疑的官方文档补充 * 问题回答的内容修补 * share link random avatar (#3541) * share link random avatar * fix * delete unused code * share page avatar (#3558) * feat: init 4818 * share page avatar * feat: tmp upgrade code (#3559) * feat: tmp upgrade code * fulltext search test * update action * full text tmp code (#3561) * full text tmp code * fix: init * fix: init * remove tmp code * remove tmp code * 4818-alpha * 4.8.18 test (#3562) * full text tmp code * fix: init * upgrade code * account log * account log * perf: dockerfile * upgrade code * chore: update docs app template submission (#3564) --------- Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com> Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
This commit is contained in:
Archer
123
projects/app/src/pages/api/admin/initv4818.ts
Normal file
123
projects/app/src/pages/api/admin/initv4818.ts
Normal file
@@ -0,0 +1,123 @@
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { delay } from '@fastgpt/global/common/system/utils';
|
||||
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
|
||||
import { jiebaSplit } from '@fastgpt/service/common/string/jieba';
|
||||
import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { NextApiRequest, NextApiResponse } from 'next';
|
||||
|
||||
/*
|
||||
简单版迁移：直接升级到最新镜像，会去除 MongoDatasetData 里的索引。直接执行这个脚本。
|
||||
无缝迁移：
|
||||
1. 先用 4.8.18-tmp 版本，会同时有 MongoDatasetData 和 MongoDatasetDataText 两个表和索引，依然是 MongoDatasetData 生效。会同步更新两张表数据。
|
||||
2. 执行升级脚本，不要删除 MongoDatasetData 里的数据。
|
||||
3. 切换正式版镜像，让 MongoDatasetDataText 生效。
|
||||
4. 删除 MongoDatasetData 里的索引和多余字段。（4819 再删
|
||||
*/
|
||||
let success = 0;
|
||||
async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
await authCert({ req, authRoot: true });
|
||||
|
||||
const batchSize = req.body.batchSize || 500;
|
||||
success = 0;
|
||||
|
||||
const start = Date.now();
|
||||
await initData(batchSize);
|
||||
// await restore();
|
||||
console.log('Init data time:', Date.now() - start);
|
||||
|
||||
success = 0;
|
||||
|
||||
// batchUpdateFields();
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
export default NextAPI(handler);
|
||||
|
||||
const restore = async () => {
|
||||
try {
|
||||
const data = await MongoDatasetData.findOne({ fullTextToken: { $exists: false } });
|
||||
if (!data) return;
|
||||
|
||||
data.fullTextToken = jiebaSplit({ text: `${data.q}\n${data.a}`.trim() });
|
||||
await data.save();
|
||||
|
||||
success++;
|
||||
console.log('Success:', success);
|
||||
|
||||
await restore();
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
await delay(500);
|
||||
await restore();
|
||||
}
|
||||
};
|
||||
|
||||
const initData = async (batchSize: number) => {
|
||||
while (true) {
|
||||
try {
|
||||
// 找到没有初始化的数据
|
||||
const dataList = await MongoDatasetData.find(
|
||||
{
|
||||
initFullText: { $exists: false }
|
||||
},
|
||||
'_id teamId datasetId collectionId fullTextToken'
|
||||
)
|
||||
.limit(batchSize)
|
||||
.lean();
|
||||
|
||||
if (dataList.length === 0) break;
|
||||
|
||||
try {
|
||||
await MongoDatasetDataText.insertMany(
|
||||
dataList.map((item) => ({
|
||||
teamId: item.teamId,
|
||||
datasetId: item.datasetId,
|
||||
collectionId: item.collectionId,
|
||||
dataId: item._id,
|
||||
fullTextToken: item.fullTextToken
|
||||
})),
|
||||
{ ordered: false, lean: true }
|
||||
);
|
||||
} catch (error: any) {
|
||||
if (error.code === 11000) {
|
||||
console.log('Duplicate key error');
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 把成功插入的新数据的 dataId 更新为已初始化
|
||||
await MongoDatasetData.updateMany(
|
||||
{ _id: { $in: dataList.map((item) => item._id) } },
|
||||
// FullText tmp
|
||||
// { $set: { initFullText: true } }
|
||||
{ $set: { initFullText: true }, $unset: { fullTextToken: 1 } }
|
||||
);
|
||||
|
||||
success += dataList.length;
|
||||
console.log('Success:', success);
|
||||
|
||||
// await initData(batchSize);
|
||||
} catch (error: any) {
|
||||
console.log(error, '===');
|
||||
await delay(500);
|
||||
// await initData(batchSize);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const batchUpdateFields = async (batchSize = 2000) => {
|
||||
// Update in batches
|
||||
await MongoDatasetData.updateMany(
|
||||
{ initFullText: { $exists: true } },
|
||||
{
|
||||
$unset: {
|
||||
initFullText: 1,
|
||||
fullTextToken: 1
|
||||
}
|
||||
}
|
||||
);
|
||||
};
|
||||
Reference in New Issue
Block a user