Collection tag (#2266)

* feat: collection metadata filter (#2211)

* feat: add dataset collection tags (#2231)

* dataset page

* workflow page

* move

* fix

* add plus filter

* fix

* fix

* fix

* perf: collection tag code

* fix: collection tags (#2249)

* fix

* fix

* fix tags of dataset page

* fix tags of workflow page

* doc

* add comments

* fix: collection tags (#2264)

* fix: metadata filter

* feat: search filter

---------

Co-authored-by: heheer <1239331448@qq.com>
Co-authored-by: heheer <heheer@sealos.io>

projects/app/src/pages/api/core/dataset/collection/list.ts

@@ -20,6 +20,7 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
     parentId = null,
     searchText = '',
     selectFolder = false,
+    filterTags = [],
     simple = false
   } = req.body as GetDatasetCollectionsProps;
   searchText = searchText?.replace(/'/g, '');
@@ -43,7 +44,8 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
       ? {
           name: new RegExp(searchText, 'i')
         }
-      : {})
+      : {}),
+    ...(filterTags.length ? { tags: { $in: filterTags } } : {})
   };
 
   const selectField = {
@@ -57,7 +59,8 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
     updateTime: 1,
     trainingType: 1,
     fileId: 1,
-    rawLink: 1
+    rawLink: 1,
+    tags: 1
   };
 
   // not count data amount
@@ -68,6 +71,7 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
         updateTime: -1
       })
       .lean();
+
     return {
       pageNum,
       pageSize,

projects/app/src/pages/api/core/dataset/collection/scrollList.ts

@@ -0,0 +1,191 @@
+import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
+import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { NextAPI } from '@/service/middleware/entry';
+import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema';
+import { Types } from '@fastgpt/service/common/mongo';
+import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
+import { startTrainingQueue } from '@/service/core/dataset/training/utils';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
+import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
+import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
+
+export type GetScrollCollectionsProps = PaginationProps<{
+  datasetId: string;
+  parentId?: string | null;
+  searchText?: string;
+  selectFolder?: boolean;
+  filterTags?: string[];
+  simple?: boolean;
+}>;
+
+async function handler(
+  req: ApiRequestProps<{}, GetScrollCollectionsProps>
+): Promise<PaginationResponse<DatasetCollectionsListItemType>> {
+  let {
+    datasetId,
+    pageSize = 10,
+    current = 1,
+    parentId = null,
+    searchText = '',
+    selectFolder = false,
+    filterTags = [],
+    simple = false
+  } = req.query;
+  if (!datasetId) {
+    return Promise.reject(CommonErrEnum.missingParams);
+  }
+  searchText = searchText?.replace(/'/g, '');
+  pageSize = Math.min(pageSize, 30);
+
+  // auth dataset and get my role
+  const { teamId, permission } = await authDataset({
+    req,
+    authToken: true,
+    authApiKey: true,
+    datasetId,
+    per: ReadPermissionVal
+  });
+
+  const match = {
+    teamId: new Types.ObjectId(teamId),
+    datasetId: new Types.ObjectId(datasetId),
+    parentId: parentId ? new Types.ObjectId(parentId) : null,
+    ...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}),
+    ...(searchText
+      ? {
+          name: new RegExp(searchText, 'i')
+        }
+      : {}),
+    ...(filterTags.length ? { tags: { $all: filterTags } } : {})
+  };
+
+  const selectField = {
+    _id: 1,
+    parentId: 1,
+    tmbId: 1,
+    name: 1,
+    type: 1,
+    forbid: 1,
+    createTime: 1,
+    updateTime: 1,
+    trainingType: 1,
+    fileId: 1,
+    rawLink: 1,
+    tags: 1
+  };
+
+  // not count data amount
+  if (simple) {
+    const collections = await MongoDatasetCollection.find(match)
+      .select(selectField)
+      .sort({
+        updateTime: -1
+      })
+      .skip(pageSize * (current - 1))
+      .limit(pageSize)
+      .lean();
+
+    return {
+      list: await Promise.all(
+        collections.map(async (item) => ({
+          ...item,
+          dataAmount: 0,
+          trainingAmount: 0,
+          permission
+        }))
+      ),
+      total: await MongoDatasetCollection.countDocuments(match)
+    };
+  }
+
+  const [collections, total]: [DatasetCollectionsListItemType[], number] = await Promise.all([
+    MongoDatasetCollection.aggregate([
+      {
+        $match: match
+      },
+      {
+        $sort: { updateTime: -1 }
+      },
+      {
+        $skip: (current - 1) * pageSize
+      },
+      {
+        $limit: pageSize
+      },
+      // count training data
+      {
+        $lookup: {
+          from: DatasetTrainingCollectionName,
+          let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId },
+          pipeline: [
+            {
+              $match: {
+                $expr: {
+                  $and: [{ $eq: ['$teamId', '$$team_id'] }, { $eq: ['$collectionId', '$$id'] }]
+                }
+              }
+            },
+            { $count: 'count' }
+          ],
+          as: 'trainingCount'
+        }
+      },
+      // count collection total data
+      {
+        $lookup: {
+          from: DatasetDataCollectionName,
+          let: { id: '$_id', team_id: match.teamId, dataset_id: match.datasetId },
+          pipeline: [
+            {
+              $match: {
+                $expr: {
+                  $and: [
+                    { $eq: ['$teamId', '$$team_id'] },
+                    { $eq: ['$datasetId', '$$dataset_id'] },
+                    { $eq: ['$collectionId', '$$id'] }
+                  ]
+                }
+              }
+            },
+            { $count: 'count' }
+          ],
+          as: 'dataCount'
+        }
+      },
+      {
+        $project: {
+          ...selectField,
+          dataAmount: {
+            $ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0]
+          },
+          trainingAmount: {
+            $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0]
+          }
+        }
+      }
+    ]),
+    MongoDatasetCollection.countDocuments(match)
+  ]);
+
+  const data = await Promise.all(
+    collections.map(async (item) => ({
+      ...item,
+      permission
+    }))
+  );
+
+  if (data.find((item) => item.trainingAmount > 0)) {
+    startTrainingQueue();
+  }
+
+  // count collections
+  return {
+    list: data,
+    total
+  };
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/collection/update.ts

@@ -10,11 +10,12 @@ export type UpdateDatasetCollectionParams = {
   id: string;
   parentId?: string;
   name?: string;
+  tags?: string[];
   forbid?: boolean;
 };
 
 async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
-  const { id, parentId, name, forbid } = req.body;
+  const { id, parentId, name, tags, forbid } = req.body;
 
   if (!id) {
     return Promise.reject(CommonErrEnum.missingParams);
@@ -32,6 +33,7 @@ async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
   const updateFields: Record<string, any> = {
     ...(parentId !== undefined && { parentId: parentId || null }),
     ...(name && { name, updateTime: getCollectionUpdateTime({ name }) }),
+    ...(tags && { tags }),
     ...(forbid !== undefined && { forbid })
   };