Add externalfile api params (#2745)

* feat: external dataset api

* perf: doc

projects/app/src/pages/api/core/dataset/collection/detail.ts

@@ -10,6 +10,7 @@ import { NextAPI } from '@/service/middleware/entry';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils';
 
 async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType> {
   const { id } = req.query as { id: string };
@@ -35,6 +36,10 @@ async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType>
   return {
     ...collection,
     ...getCollectionSourceData(collection),
+    tags: await collectionTagsToTagLabel({
+      datasetId: collection.datasetId._id,
+      tags: collection.tags
+    }),
     permission,
     file
   };

projects/app/src/pages/api/core/dataset/collection/list.ts

@@ -11,6 +11,8 @@ import { startTrainingQueue } from '@/service/core/dataset/training/utils';
 import { NextAPI } from '@/service/middleware/entry';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { PagingData } from '@/types';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
+import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils';
 
 async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectionsListItemType>> {
   let {
@@ -60,12 +62,15 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
     trainingType: 1,
     fileId: 1,
     rawLink: 1,
-    tags: 1
+    tags: 1,
+    externalFileId: 1
   };
 
   // not count data amount
   if (simple) {
-    const collections = await MongoDatasetCollection.find(match)
+    const collections = await MongoDatasetCollection.find(match, undefined, {
+      ...readFromSecondary
+    })
       .select(selectField)
       .sort({
         updateTime: -1
@@ -78,6 +83,10 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
       data: await Promise.all(
         collections.map(async (item) => ({
           ...item,
+          tags: await collectionTagsToTagLabel({
+            datasetId,
+            tags: item.tags
+          }),
           dataAmount: 0,
           trainingAmount: 0,
           permission
@@ -153,12 +162,18 @@ async function handler(req: NextApiRequest): Promise<PagingData<DatasetCollectio
         }
       }
     ]),
-    MongoDatasetCollection.countDocuments(match)
+    MongoDatasetCollection.countDocuments(match, {
+      ...readFromSecondary
+    })
   ]);
 
   const data = await Promise.all(
     collections.map(async (item) => ({
       ...item,
+      tags: await collectionTagsToTagLabel({
+        datasetId,
+        tags: item.tags
+      }),
       permission
     }))
   );

projects/app/src/pages/api/core/dataset/collection/update.ts

@@ -1,5 +1,8 @@
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
-import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
+import {
+  createOrGetCollectionTags,
+  getCollectionUpdateTime
+} from '@fastgpt/service/core/dataset/collection/utils';
 import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
 import { NextAPI } from '@/service/middleware/entry';
 import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
@@ -11,11 +14,16 @@ import { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
 
 export type UpdateDatasetCollectionParams = {
-  id: string;
+  id?: string;
   parentId?: string;
   name?: string;
-  tags?: string[];
+  tags?: string[]; // Not tag id, is tag label
   forbid?: boolean;
+  createTime?: Date;
+
+  // External file id
+  datasetId?: string;
+  externalFileId?: string;
 };
 
 // Set folder collection children forbid status
@@ -65,14 +73,22 @@ const updateFolderChildrenForbid = async ({
 };
 
 async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
-  const { id, parentId, name, tags, forbid } = req.body;
+  let { datasetId, externalFileId, id, parentId, name, tags, forbid, createTime } = req.body;
+
+  if (datasetId && externalFileId) {
+    const collection = await MongoDatasetCollection.findOne({ datasetId, externalFileId }, '_id');
+    if (!collection) {
+      return Promise.reject(CommonErrEnum.fileNotFound);
+    }
+    id = collection._id;
+  }
 
   if (!id) {
     return Promise.reject(CommonErrEnum.missingParams);
   }
 
   // 凭证校验
-  const { collection } = await authDatasetCollection({
+  const { collection, teamId } = await authDatasetCollection({
     req,
     authToken: true,
     authApiKey: true,
@@ -81,6 +97,13 @@ async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
   });
 
   await mongoSessionRun(async (session) => {
+    const collectionTags = await createOrGetCollectionTags({
+      tags,
+      teamId,
+      datasetId: collection.datasetId._id,
+      session
+    });
+
     await MongoDatasetCollection.updateOne(
       {
         _id: id
@@ -89,8 +112,9 @@ async function handler(req: ApiRequestProps<UpdateDatasetCollectionParams>) {
         $set: {
           ...(parentId !== undefined && { parentId: parentId || null }),
           ...(name && { name, updateTime: getCollectionUpdateTime({ name }) }),
-          ...(tags && { tags }),
-          ...(forbid !== undefined && { forbid })
+          ...(collectionTags !== undefined && { tags: collectionTags }),
+          ...(forbid !== undefined && { forbid }),
+          ...(createTime !== undefined && { createTime })
         }
       },
       {