Dataset folder manager (#274)

* feat: retry send

* perf: qa default value

* feat: dataset folder

* feat: kb folder delete and path

* fix: ts

* perf: script load

* feat: fileCard and dataCard

* feat: search file

* feat: max token

* feat: select dataset

* fix: preview chunk

* perf: source update

* export data limit file_id

* docs

* fix: export limit

client/src/pages/api/plugins/kb/delete.ts

@@ -3,12 +3,12 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase, KB, App, TrainingData } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { PgClient } from '@/service/pg';
-import { Types } from 'mongoose';
 import { PgTrainingTableName } from '@/constants/plugin';
 import { GridFSStorage } from '@/service/lib/gridfs';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const { id } = req.query as {
       id: string;
     };
@@ -20,26 +20,30 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
+    const deletedIds = [id, ...(await findAllChildrenIds(id))];
 
     // delete training data
     await TrainingData.deleteMany({
       userId,
-      kbId: id
+      kbId: { $in: deletedIds }
     });
 
     // delete all pg data
     await PgClient.delete(PgTrainingTableName, {
-      where: [['user_id', userId], 'AND', ['kb_id', id]]
+      where: [
+        ['user_id', userId],
+        'AND',
+        `kb_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`
+      ]
     });
 
     // delete related files
     const gridFs = new GridFSStorage('dataset', userId);
-    await gridFs.deleteFilesByKbId(id);
+    await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id)));
 
     // delete kb data
-    await KB.findOneAndDelete({
-      _id: id,
+    await KB.deleteMany({
+      _id: { $in: deletedIds },
       userId
     });
 
@@ -51,3 +55,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
   }
 }
+
+async function findAllChildrenIds(id: string) {
+  // find children
+  const children = await KB.find({ parentId: id });
+
+  let allChildrenIds = children.map((child) => String(child._id));
+
+  for (const child of children) {
+    const grandChildrenIds = await findAllChildrenIds(child._id);
+    allChildrenIds = allChildrenIds.concat(grandChildrenIds);
+  }
+
+  return allChildrenIds;
+}