feat: sync org from wecom, pref: member list pagination (#3549)

* feat: sync org

* chore: fe

* chore: loading

* chore: type

* pref: team member list change to pagination. Edit a sort of list apis.

* feat: member update avatar

* chore: user avatar move to tmb

* chore: init scripts move user avatar

* chore: sourceMember

* fix: list api sourceMember

* fix: member sync

* fix: pagination

* chore: adjust code

* chore: move changeOwner to pro

* chore: init v4819 script

* chore: adjust code

* chore: UserBox
This commit is contained in:
Finley Ge
2025-01-13 11:22:20 +08:00
committed by archer
parent a8d456f448
commit ec0cef09a2
73 changed files with 883 additions and 757 deletions

View File

@@ -5,6 +5,8 @@ import { jiebaSplit } from '@fastgpt/service/common/string/jieba';
import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
import { NextApiRequest, NextApiResponse } from 'next';
/*
@@ -14,6 +16,7 @@ import { NextApiRequest, NextApiResponse } from 'next';
2. 执行升级脚本,不要删除 MongoDatasetData 里的数据。
3. 切换正式版镜像,让 MongoDatasetDataText 生效。
4. 删除 MongoDatasetData 里的索引和多余字段。4819 再删
5. 移动 User 表中的 avatar 字段到 TeamMember 表中。
*/
let success = 0;
async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -109,15 +112,26 @@ const initData = async (batchSize: number) => {
}
};
const batchUpdateFields = async (batchSize = 2000) => {
// Update in batches
await MongoDatasetData.updateMany(
{ initFullText: { $exists: true } },
{
$unset: {
initFullText: 1,
fullTextToken: 1
}
}
);
};
// const batchUpdateFields = async (batchSize = 2000) => {
// // Find documents that still have these fields
// const documents = await MongoDatasetData.find({ initFullText: { $exists: true } }, '_id')
// .limit(batchSize)
// .lean();
// if (documents.length === 0) return;
// // Update in batches
// await MongoDatasetData.updateMany(
// { _id: { $in: documents.map((doc) => doc._id) } },
// {
// $unset: {
// initFullText: 1
// // fullTextToken: 1
// }
// }
// );
// success += documents.length;
// console.log('Delete success:', success);
// await batchUpdateFields(batchSize);
// };

View File

@@ -0,0 +1,37 @@
import { NextAPI } from '@/service/middleware/entry';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
import { NextApiRequest, NextApiResponse } from 'next';
/*
简单版迁移:直接升级到最新镜像,会去除 MongoDatasetData 里的索引。直接执行这个脚本。
无缝迁移:
1. 移动 User 表中的 avatar 字段到 TeamMember 表中。
*/
async function handler(req: NextApiRequest, res: NextApiResponse) {
await authCert({ req, authRoot: true });
await moveUserAvatar();
return { success: true };
}
export default NextAPI(handler);
const moveUserAvatar = async () => {
try {
const users = await MongoUser.find({});
for await (const user of users) {
await MongoTeamMember.updateOne(
{
_id: user._id
},
{
avatar: (user as any).avatar // 删除 avatar 字段, 因为 Type 改了,所以这里不能直接写 user.avatar
}
);
}
console.log('Move avatar success:', users.length);
} catch (error) {
console.error(error);
}
};

View File

@@ -1,6 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import type { PagingData } from '@/types';
import { AppLogsListItemType } from '@/types/app';
import { Types } from '@fastgpt/service/common/mongo';
import { addDays } from 'date-fns';
@@ -10,19 +9,22 @@ import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchem
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant';
import { PaginationResponse } from '@fastgpt/web/common/fetch/type';
async function handler(
req: NextApiRequest,
_res: NextApiResponse
): Promise<PagingData<AppLogsListItemType>> {
): Promise<PaginationResponse<AppLogsListItemType>> {
const {
pageNum = 1,
pageSize = 20,
appId,
dateStart = addDays(new Date(), -7),
dateEnd = new Date()
} = req.body as GetAppChatLogsParams;
const { pageSize = 20, offset } = parsePaginationRequest(req);
if (!appId) {
throw new Error('缺少参数');
}
@@ -39,7 +41,7 @@ async function handler(
}
};
const [data, total] = await Promise.all([
const [list, total] = await Promise.all([
MongoChat.aggregate(
[
{ $match: where },
@@ -51,7 +53,7 @@ async function handler(
updateTime: -1
}
},
{ $skip: (pageNum - 1) * pageSize },
{ $skip: offset },
{ $limit: pageSize },
{
$lookup: {
@@ -80,6 +82,14 @@ async function handler(
as: 'chatitems'
}
},
{
$lookup: {
from: TeamMemberCollectionName,
localField: 'tmbId',
foreignField: '_id',
as: 'member'
}
},
{
$addFields: {
userGoodFeedbackCount: {
@@ -133,7 +143,12 @@ async function handler(
customFeedbacksCount: 1,
markCount: 1,
outLinkUid: 1,
tmbId: 1
tmbId: 1,
sourceMember: {
name: '$member.name',
avatar: '$member.avatar',
status: '$member.status'
}
}
}
],
@@ -145,9 +160,7 @@ async function handler(
]);
return {
pageNum,
pageSize,
data,
list,
total
};
}

View File

@@ -18,6 +18,7 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import { concatPer } from '@fastgpt/service/support/permission/controller';
import { getGroupsByTmbId } from '@fastgpt/service/support/permission/memberGroup/controllers';
import { getOrgIdSetWithParentByTmbId } from '@fastgpt/service/support/permission/org/controllers';
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
export type ListAppBody = {
parentId?: ParentIdType;
@@ -201,19 +202,33 @@ async function handler(req: ApiRequestProps<ListAppBody>): Promise<AppListItemTy
})
.filter((app) => app.permission.hasReadPer);
return formatApps.map((app) => ({
_id: app._id,
tmbId: app.tmbId,
avatar: app.avatar,
type: app.type,
name: app.name,
intro: app.intro,
updateTime: app.updateTime,
permission: app.permission,
pluginData: app.pluginData,
inheritPermission: app.inheritPermission ?? true,
private: app.privateApp
}));
// get member info
const memberInfo = await MongoTeamMember.find(
{ _id: { $in: formatApps.map((app) => app.tmbId) } },
'_id name avatar status'
).lean();
return formatApps.map((app) => {
const member = memberInfo.find((item) => String(item._id) === String(app.tmbId))!;
return {
_id: app._id,
tmbId: app.tmbId,
avatar: app.avatar,
type: app.type,
name: app.name,
intro: app.intro,
updateTime: app.updateTime,
permission: app.permission,
pluginData: app.pluginData,
inheritPermission: app.inheritPermission ?? true,
private: app.privateApp,
sourceMember: {
name: member.name,
avatar: member.avatar,
status: member.status
}
};
});
}
export default NextAPI(handler);

View File

@@ -6,6 +6,8 @@ import { ApiRequestProps } from '@fastgpt/service/type/next';
import { authApp } from '@fastgpt/service/support/permission/app/auth';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { VersionListItemType } from '@fastgpt/global/core/app/version';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
export type versionListBody = PaginationProps<{
appId: string;
@@ -15,24 +17,44 @@ export type versionListResponse = PaginationResponse<VersionListItemType>;
async function handler(
req: ApiRequestProps<versionListBody>,
res: NextApiResponse<any>
_res: NextApiResponse<any>
): Promise<versionListResponse> {
const { offset, pageSize, appId } = req.body;
const { appId } = req.body;
const { offset, pageSize } = parsePaginationRequest(req);
await authApp({ appId, req, per: WritePermissionVal, authToken: true });
const [result, total] = await Promise.all([
MongoAppVersion.find(
{
(async () => {
const versions = await MongoAppVersion.find({
appId
},
'_id appId versionName time isPublish tmbId'
)
.sort({
time: -1
})
.skip(offset)
.limit(pageSize),
.sort({
time: -1
})
.skip(offset)
.limit(pageSize)
.lean();
const memberList = await MongoTeamMember.find(
{
_id: { $in: versions.map((item) => item.tmbId) }
},
'_id name avatar status'
).lean();
return versions.map((item) => {
const member = memberList.find((member) => String(member._id) === String(item.tmbId));
return {
...item,
sourceMember: {
name: member?.name || '',
avatar: member?.avatar || '',
status: member?.status || ''
}
};
});
})(),
MongoAppVersion.countDocuments({ appId })
]);
@@ -43,7 +65,8 @@ async function handler(
versionName: item.versionName,
time: item.time,
isPublish: item.isPublish,
tmbId: item.tmbId
tmbId: item.tmbId,
sourceMember: item.sourceMember
};
});

View File

@@ -12,7 +12,6 @@ describe('发布应用版本测试', () => {
nodes: [],
edges: [],
chatConfig: {},
type: AppTypeEnum.simple,
isPublish: false,
versionName: '1'
};

View File

@@ -7,6 +7,7 @@ import { NextAPI } from '@/service/middleware/entry';
import { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import { GetHistoriesProps } from '@/global/core/chat/api';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { addMonths } from 'date-fns';
export type getHistoriesQuery = {};
@@ -17,9 +18,10 @@ export type getHistoriesResponse = {};
async function handler(
req: ApiRequestProps<getHistoriesBody, getHistoriesQuery>,
res: ApiResponseType<any>
_res: ApiResponseType<any>
): Promise<PaginationResponse<getHistoriesResponse>> {
const { appId, shareId, outLinkUid, teamId, teamToken, offset, pageSize, source } = req.body;
const { appId, shareId, outLinkUid, teamId, teamToken, source } = req.body;
const { offset, pageSize } = parsePaginationRequest(req);
const match = await (async () => {
if (shareId && outLinkUid) {

View File

@@ -13,6 +13,7 @@ import { filterPublicNodeResponseData } from '@fastgpt/global/core/chat/utils';
import { GetChatTypeEnum } from '@/global/core/chat/constants';
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import { ChatItemType } from '@fastgpt/global/core/chat/type';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
export type getPaginationRecordsQuery = {};
@@ -22,16 +23,11 @@ export type getPaginationRecordsResponse = PaginationResponse<ChatItemType>;
async function handler(
req: ApiRequestProps<getPaginationRecordsBody, getPaginationRecordsQuery>,
res: ApiResponseType<any>
_res: ApiResponseType<any>
): Promise<getPaginationRecordsResponse> {
const {
appId,
chatId,
offset,
pageSize = 10,
loadCustomFeedbacks,
type = GetChatTypeEnum.normal
} = req.body;
const { appId, chatId, loadCustomFeedbacks, type = GetChatTypeEnum.normal } = req.body;
const { offset, pageSize } = parsePaginationRequest(req);
if (!appId || !chatId) {
return {

View File

@@ -6,6 +6,7 @@ import { ApiRequestProps } from '@fastgpt/service/type/next';
import { ChatInputGuideSchemaType } from '@fastgpt/global/core/chat/inputGuide/type';
import { authApp } from '@fastgpt/service/support/permission/app/auth';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
export type ChatInputGuideProps = PaginationProps<{
appId: string;
@@ -17,7 +18,8 @@ async function handler(
req: ApiRequestProps<ChatInputGuideProps>,
res: NextApiResponse<any>
): Promise<ChatInputGuideResponse> {
const { appId, pageSize, offset, searchKey } = req.body;
const { appId, searchKey } = req.body;
const { offset, pageSize } = parsePaginationRequest(req);
await authApp({ req, appId, authToken: true, per: ReadPermissionVal });

View File

@@ -10,14 +10,15 @@ import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/sc
import { startTrainingQueue } from '@/service/core/dataset/training/utils';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { PagingData } from '@/types';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils';
import { PaginationResponse } from '@fastgpt/web/common/fetch/type';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectionsListItemType>> {
async function handler(
req: NextApiRequest
): Promise<PaginationResponse<DatasetCollectionsListItemType>> {
let {
pageNum = 1,
pageSize = 10,
datasetId,
parentId = null,
searchText = '',
@@ -25,8 +26,9 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
filterTags = [],
simple = false
} = req.body as GetDatasetCollectionsProps;
searchText = searchText?.replace(/'/g, '');
let { pageSize, offset } = parsePaginationRequest(req);
pageSize = Math.min(pageSize, 30);
searchText = searchText?.replace(/'/g, '');
// auth dataset and get my role
const { teamId, permission } = await authDataset({
@@ -78,9 +80,7 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
.lean();
return {
pageNum,
pageSize,
data: await Promise.all(
list: await Promise.all(
collections.map(async (item) => ({
...item,
tags: await collectionTagsToTagLabel({
@@ -105,7 +105,7 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
$sort: { updateTime: -1 }
},
{
$skip: (pageNum - 1) * pageSize
$skip: offset
},
{
$limit: pageSize
@@ -167,7 +167,7 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
})
]);
const data = await Promise.all(
const list = await Promise.all(
collections.map(async (item) => ({
...item,
tags: await collectionTagsToTagLabel({
@@ -178,15 +178,13 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
}))
);
if (data.find((item) => item.trainingAmount > 0)) {
if (list.find((item) => item.trainingAmount > 0)) {
startTrainingQueue();
}
// count collections
return {
pageNum,
pageSize,
data,
list,
total
};
}

View File

@@ -10,6 +10,7 @@ import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
export type GetScrollCollectionsProps = PaginationProps<{
datasetId: string;
@@ -25,8 +26,6 @@ async function handler(
): Promise<PaginationResponse<DatasetCollectionsListItemType>> {
let {
datasetId,
pageSize = 10,
offset,
parentId = null,
searchText = '',
selectFolder = false,
@@ -36,6 +35,7 @@ async function handler(
if (!datasetId) {
return Promise.reject(CommonErrEnum.missingParams);
}
let { offset, pageSize } = parsePaginationRequest(req);
searchText = searchText?.replace(/'/g, '');
pageSize = Math.min(pageSize, 30);

View File

@@ -3,19 +3,21 @@ import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { PagingData, RequestPaging } from '@/types';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { DatasetDataListItemType } from '@/global/core/dataset/type';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { PaginationResponse } from '@fastgpt/web/common/fetch/type';
export type GetDatasetDataListProps = RequestPaging & {
export type GetDatasetDataListProps = {
searchText?: string;
collectionId: string;
};
async function handler(
req: ApiRequestProps<GetDatasetDataListProps>
): Promise<PagingData<DatasetDataListItemType>> {
let { pageNum = 1, pageSize = 10, searchText = '', collectionId } = req.body;
): Promise<PaginationResponse<DatasetDataListItemType>> {
let { searchText = '', collectionId } = req.body;
let { offset, pageSize } = parsePaginationRequest(req);
pageSize = Math.min(pageSize, 30);
@@ -40,19 +42,17 @@ async function handler(
: {})
};
const [data, total] = await Promise.all([
const [list, total] = await Promise.all([
MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex')
.sort({ chunkIndex: 1, updateTime: -1 })
.skip((pageNum - 1) * pageSize)
.skip(offset)
.limit(pageSize)
.lean(),
MongoDatasetData.countDocuments(match)
]);
return {
pageNum,
pageSize,
data,
list,
total
};
}

View File

@@ -6,6 +6,7 @@ import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { DatasetDataListItemType } from '@/global/core/dataset/type';
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
export type GetDatasetDataListProps = PaginationProps & {
searchText?: string;
@@ -16,7 +17,8 @@ export type GetDatasetDataListRes = PaginationResponse<DatasetDataListItemType>;
async function handler(
req: ApiRequestProps<GetDatasetDataListProps>
): Promise<GetDatasetDataListRes> {
let { offset, pageSize = 10, searchText = '', collectionId } = req.body;
let { searchText = '', collectionId } = req.body;
let { offset, pageSize } = parsePaginationRequest(req);
pageSize = Math.min(pageSize, 30);

View File

@@ -2,7 +2,6 @@ import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { authUserPer } from '@fastgpt/service/support/permission/user/auth';
import { getVectorModel } from '@fastgpt/service/core/ai/model';
import { NextAPI } from '@/service/middleware/entry';
import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
import {
@@ -19,6 +18,7 @@ import { replaceRegChars } from '@fastgpt/global/common/string/tools';
import { getGroupsByTmbId } from '@fastgpt/service/support/permission/memberGroup/controllers';
import { concatPer } from '@fastgpt/service/support/permission/controller';
import { getOrgIdSetWithParentByTmbId } from '@fastgpt/service/support/permission/org/controllers';
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
export type GetDatasetListBody = {
parentId: ParentIdType;
@@ -174,19 +174,20 @@ async function handler(req: ApiRequestProps<GetDatasetListBody>) {
})
.filter((app) => app.permission.hasReadPer);
const data = formatDatasets.map<DatasetListItemType>((item) => ({
_id: item._id,
avatar: item.avatar,
name: item.name,
intro: item.intro,
type: item.type,
permission: item.permission,
vectorModel: getVectorModel(item.vectorModel),
inheritPermission: item.inheritPermission,
tmbId: item.tmbId,
updateTime: item.updateTime,
private: item.privateDataset
}));
const tmbIds = formatDatasets.map((item) => item.tmbId);
const memberInfo = await MongoTeamMember.find({ _id: { $in: tmbIds } }, '_id name avatar').lean();
const data = formatDatasets.map((item) => {
const member = memberInfo.find((member) => String(member._id) === String(item.tmbId));
return {
...item,
sourceMember: {
name: member!.name,
avatar: member!.avatar
}
};
});
return data;
}

View File

@@ -1,17 +1,18 @@
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { UserUpdateParams } from '@/types/user';
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
/* update user info */
import type { ApiRequestProps, ApiResponseType } from '@fastgpt/service/type/next';
import { NextAPI } from '@/service/middleware/entry';
import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
import { getUserDetail } from '@fastgpt/service/support/user/controller';
import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller';
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
export type UserAccountUpdateQuery = {};
export type UserAccountUpdateBody = UserUpdateParams;
export type UserAccountUpdateResponse = {};
async function handler(
req: ApiRequestProps<UserAccountUpdateBody, UserAccountUpdateQuery>,
_res: ApiResponseType<any>
@@ -19,21 +20,33 @@ async function handler(
const { avatar, timezone } = req.body;
const { tmbId } = await authCert({ req, authToken: true });
const user = await getUserDetail({ tmbId });
// const user = await getUserDetail({ tmbId });
// 更新对应的记录
await mongoSessionRun(async (session) => {
await MongoUser.updateOne(
{
_id: user._id
},
{
...(avatar && { avatar }),
...(timezone && { timezone })
}
).session(session);
await refreshSourceAvatar(avatar, user.avatar, session);
const tmb = await MongoTeamMember.findById(tmbId).session(session);
if (timezone) {
await MongoUser.updateOne(
{
_id: tmb?.userId
},
{
timezone
}
).session(session);
}
// if avatar, update team member avatar
if (avatar) {
await MongoTeamMember.updateOne(
{
_id: tmbId
},
{
avatar
}
).session(session);
await refreshSourceAvatar(avatar, tmb?.avatar, session);
}
});
return {};