v4.6.9-alpha (#918)

Co-authored-by: Mufei <327958099@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

projects/app/src/pages/api/timerTask/dataset/checkInvalidDatasetImage.ts

@@ -0,0 +1,88 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@fastgpt/service/common/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { addLog } from '@fastgpt/service/common/system/log';
+import { addHours } from 'date-fns';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { MongoImage } from '@fastgpt/service/common/file/image/schema';
+
+/* 
+  检测无效的数据集图片
+
+  可能异常情况：
+  1. 上传文件过程中，上传了图片，但是最终没有创建数据集。
+*/
+
+let deleteImageAmount = 0;
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const {
+      startHour = 72,
+      endHour = 24,
+      limit = 10
+    } = req.body as { startHour?: number; endHour?: number; limit?: number };
+    await authCert({ req, authRoot: true });
+    await connectToDatabase();
+
+    // start: now - maxDay, end: now - 3 day
+    const start = addHours(new Date(), -startHour);
+    const end = addHours(new Date(), -endHour);
+    deleteImageAmount = 0;
+
+    await checkInvalid(start, end, limit);
+
+    jsonRes(res, {
+      data: deleteImageAmount
+    });
+  } catch (error) {
+    addLog.error(`check Invalid user error`, error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+export async function checkInvalid(start: Date, end: Date, limit = 50) {
+  const images = await MongoImage.find(
+    {
+      createTime: {
+        $gte: start,
+        $lte: end
+      },
+      'metadata.relatedId': { $exists: true }
+    },
+    '_id teamId metadata'
+  );
+  console.log('total images', images.length);
+  let index = 0;
+
+  for await (const image of images) {
+    try {
+      // 1. 检测是否有对应的集合
+      const collection = await MongoDatasetCollection.findOne(
+        {
+          teamId: image.teamId,
+          'metadata.relatedImgId': image.metadata?.relatedId
+        },
+        '_id'
+      );
+
+      if (!collection) {
+        await image.deleteOne();
+        deleteImageAmount++;
+      }
+
+      index++;
+
+      index % 100 === 0 && console.log(index);
+    } catch (error) {
+      console.log(error);
+    }
+  }
+
+  console.log(`检测完成，共删除 ${deleteImageAmount} 个无效图片`);
+}