From 11cfe8a809ffa89f83c52554d162efca60bd5da7 Mon Sep 17 00:00:00 2001 From: Finley Ge <32237950+FinleyGe@users.noreply.github.com> Date: Tue, 14 Jan 2025 18:39:26 +0800 Subject: [PATCH] fix: collection list api old version (#3591) * fix: collection list api format * fix: type error of addSourceMemeber --- .../app/src/pages/api/core/app/getChatLogs.ts | 3 +- projects/app/src/pages/api/core/app/list.ts | 3 +- .../pages/api/core/dataset/collection/list.ts | 28 +-- .../api/core/dataset/collection/listV2.ts | 192 ++++++++++++++++++ projects/app/src/web/core/dataset/api.ts | 2 +- 5 files changed, 209 insertions(+), 19 deletions(-) create mode 100644 projects/app/src/pages/api/core/dataset/collection/listV2.ts diff --git a/projects/app/src/pages/api/core/app/getChatLogs.ts b/projects/app/src/pages/api/core/app/getChatLogs.ts index 164cf1429..566c44a2b 100644 --- a/projects/app/src/pages/api/core/app/getChatLogs.ts +++ b/projects/app/src/pages/api/core/app/getChatLogs.ts @@ -147,8 +147,7 @@ async function handler( ]); const listWithSourceMember = await addSourceMember({ - list: list, - teamId + list: list }); const listWithoutTmbId = list.filter((item) => !item.tmbId); diff --git a/projects/app/src/pages/api/core/app/list.ts b/projects/app/src/pages/api/core/app/list.ts index d47f857aa..da515cd43 100644 --- a/projects/app/src/pages/api/core/app/list.ts +++ b/projects/app/src/pages/api/core/app/list.ts @@ -203,8 +203,7 @@ async function handler(req: ApiRequestProps): Promise app.permission.hasReadPer); return addSourceMember({ - list: formatApps, - teamId + list: formatApps }); } diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts index 8871636ee..117e10268 100644 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ b/projects/app/src/pages/api/core/dataset/collection/list.ts @@ -2,7 +2,6 @@ import type { NextApiRequest } from 'next'; import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema'; import { Types } from '@fastgpt/service/common/mongo'; import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d'; -import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; @@ -12,23 +11,20 @@ import { NextAPI } from '@/service/middleware/entry'; import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils'; -import { PaginationResponse } from '@fastgpt/web/common/fetch/type'; -import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination'; -async function handler( - req: NextApiRequest -): Promise> { +async function handler(req: NextApiRequest) { let { + pageNum = 1, + pageSize = 10, datasetId, parentId = null, searchText = '', selectFolder = false, filterTags = [], simple = false - } = req.body as GetDatasetCollectionsProps; - let { pageSize, offset } = parsePaginationRequest(req); - pageSize = Math.min(pageSize, 30); + } = req.body as any; searchText = searchText?.replace(/'/g, ''); + pageSize = Math.min(pageSize, 30); // auth dataset and get my role const { teamId, permission } = await authDataset({ @@ -80,7 +76,9 @@ async function handler( .lean(); return { - list: await Promise.all( + pageNum, + pageSize, + data: await Promise.all( collections.map(async (item) => ({ ...item, tags: await collectionTagsToTagLabel({ @@ -105,7 +103,7 @@ async function handler( $sort: { updateTime: -1 } }, { - $skip: offset + $skip: (pageNum - 1) * pageSize }, { $limit: pageSize @@ -167,7 +165,7 @@ async function handler( }) ]); - const list = await Promise.all( + const data = await Promise.all( collections.map(async (item) => ({ ...item, tags: await collectionTagsToTagLabel({ @@ -178,13 +176,15 @@ async function handler( })) ); - if (list.find((item) => item.trainingAmount > 0)) { + if (data.find((item) => item.trainingAmount > 0)) { startTrainingQueue(); } // count collections return { - list, + pageNum, + pageSize, + data, total }; } diff --git a/projects/app/src/pages/api/core/dataset/collection/listV2.ts b/projects/app/src/pages/api/core/dataset/collection/listV2.ts new file mode 100644 index 000000000..8871636ee --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/listV2.ts @@ -0,0 +1,192 @@ +import type { NextApiRequest } from 'next'; +import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema'; +import { Types } from '@fastgpt/service/common/mongo'; +import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d'; +import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq'; +import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; +import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants'; +import { authDataset } from '@fastgpt/service/support/permission/dataset/auth'; +import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema'; +import { startTrainingQueue } from '@/service/core/dataset/training/utils'; +import { NextAPI } from '@/service/middleware/entry'; +import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant'; +import { readFromSecondary } from '@fastgpt/service/common/mongo/utils'; +import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils'; +import { PaginationResponse } from '@fastgpt/web/common/fetch/type'; +import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination'; + +async function handler( + req: NextApiRequest +): Promise> { + let { + datasetId, + parentId = null, + searchText = '', + selectFolder = false, + filterTags = [], + simple = false + } = req.body as GetDatasetCollectionsProps; + let { pageSize, offset } = parsePaginationRequest(req); + pageSize = Math.min(pageSize, 30); + searchText = searchText?.replace(/'/g, ''); + + // auth dataset and get my role + const { teamId, permission } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: ReadPermissionVal + }); + + const match = { + teamId: new Types.ObjectId(teamId), + datasetId: new Types.ObjectId(datasetId), + parentId: parentId ? new Types.ObjectId(parentId) : null, + ...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}), + ...(searchText + ? { + name: new RegExp(searchText, 'i') + } + : {}), + ...(filterTags.length ? { tags: { $in: filterTags } } : {}) + }; + + const selectField = { + _id: 1, + parentId: 1, + tmbId: 1, + name: 1, + type: 1, + forbid: 1, + createTime: 1, + updateTime: 1, + trainingType: 1, + fileId: 1, + rawLink: 1, + tags: 1, + externalFileId: 1 + }; + + // not count data amount + if (simple) { + const collections = await MongoDatasetCollection.find(match, undefined, { + ...readFromSecondary + }) + .select(selectField) + .sort({ + updateTime: -1 + }) + .lean(); + + return { + list: await Promise.all( + collections.map(async (item) => ({ + ...item, + tags: await collectionTagsToTagLabel({ + datasetId, + tags: item.tags + }), + dataAmount: 0, + trainingAmount: 0, + permission + })) + ), + total: await MongoDatasetCollection.countDocuments(match) + }; + } + + const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([ + MongoDatasetCollection.aggregate([ + { + $match: match + }, + { + $sort: { updateTime: -1 } + }, + { + $skip: offset + }, + { + $limit: pageSize + }, + // count training data + { + $lookup: { + from: DatasetTrainingCollectionName, + let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, + pipeline: [ + { + $match: { + $expr: { + $and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }] + } + } + }, + { $count: 'count' } + ], + as: 'trainingCount' + } + }, + // count collection total data + { + $lookup: { + from: DatasetDataCollectionName, + let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId }, + pipeline: [ + { + $match: { + $expr: { + $and: [ + { $eq: ['$teamId', '$$team_id'] }, + { $eq: ['$datasetId', '$$dataset_id'] }, + { $eq: ['$collectionId', '$$id'] } + ] + } + } + }, + { $count: 'count' } + ], + as: 'dataCount' + } + }, + { + $project: { + ...selectField, + dataAmount: { + $ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0] + }, + trainingAmount: { + $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0] + } + } + } + ]), + MongoDatasetCollection.countDocuments(match, { + ...readFromSecondary + }) + ]); + + const list = await Promise.all( + collections.map(async (item) => ({ + ...item, + tags: await collectionTagsToTagLabel({ + datasetId, + tags: item.tags + }), + permission + })) + ); + + if (list.find((item) => item.trainingAmount > 0)) { + startTrainingQueue(); + } + + // count collections + return { + list, + total + }; +} + +export default NextAPI(handler); diff --git a/projects/app/src/web/core/dataset/api.ts b/projects/app/src/web/core/dataset/api.ts index 422b17e22..ac40e1a02 100644 --- a/projects/app/src/web/core/dataset/api.ts +++ b/projects/app/src/web/core/dataset/api.ts @@ -107,7 +107,7 @@ export const postSearchText = (data: SearchTestProps) => /* ============================= collections ==================================== */ export const getDatasetCollections = (data: GetDatasetCollectionsProps) => - POST>(`/core/dataset/collection/list`, data); + POST>(`/core/dataset/collection/listV2`, data); export const getDatasetCollectionPathById = (parentId: string) => GET(`/core/dataset/collection/paths`, { parentId }); export const getDatasetCollectionById = (id: string) =>