4.6.7-alpha commit (#743)

Co-authored-by: Archer <545436317@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/service/common/file/gridfs/controller.ts

@@ -3,9 +3,10 @@ import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import fsp from 'fs/promises';
 import fs from 'fs';
 import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
-import { delImgByFileIdList } from '../image/controller';
+import { MongoFileSchema } from './schema';
 
 export function getGFSCollection(bucket: `${BucketNameEnum}`) {
+  MongoFileSchema;
   return connectionMongo.connection.db.collection(`${bucket}.files`);
 }
 export function getGridBucket(bucket: `${BucketNameEnum}`) {
@@ -21,6 +22,7 @@ export async function uploadFile({
   tmbId,
   path,
   filename,
+  contentType,
   metadata = {}
 }: {
   bucketName: `${BucketNameEnum}`;
@@ -28,6 +30,7 @@ export async function uploadFile({
   tmbId: string;
   path: string;
   filename: string;
+  contentType?: string;
   metadata?: Record<string, any>;
 }) {
   if (!path) return Promise.reject(`filePath is empty`);
@@ -44,7 +47,7 @@ export async function uploadFile({
 
   const stream = bucket.openUploadStream(filename, {
     metadata,
-    contentType: metadata?.contentType
+    contentType
   });
 
   // save to gridfs
@@ -96,40 +99,6 @@ export async function delFileByFileIdList({
     }
   }
 }
-// delete file by metadata(datasetId)
-export async function delFileByMetadata({
-  bucketName,
-  datasetId
-}: {
-  bucketName: `${BucketNameEnum}`;
-  datasetId?: string;
-}) {
-  const bucket = getGridBucket(bucketName);
-
-  const files = await bucket
-    .find(
-      {
-        ...(datasetId && { 'metadata.datasetId': datasetId })
-      },
-      {
-        projection: {
-          _id: 1
-        }
-      }
-    )
-    .toArray();
-
-  const idList = files.map((item) => String(item._id));
-
-  // delete img
-  await delImgByFileIdList(idList);
-
-  // delete file
-  await delFileByFileIdList({
-    bucketName,
-    fileIdList: idList
-  });
-}
 
 export async function getDownloadStream({
   bucketName,

packages/service/common/file/gridfs/schema.ts

@@ -0,0 +1,15 @@
+import { connectionMongo, type Model } from '../../mongo';
+const { Schema, model, models } = connectionMongo;
+
+const FileSchema = new Schema({});
+
+try {
+  FileSchema.index({ 'metadata.teamId': 1 });
+  FileSchema.index({ 'metadata.uploadDate': -1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoFileSchema = models['dataset.files'] || model('dataset.files', FileSchema);
+
+MongoFileSchema.syncIndexes();

packages/service/common/file/image/controller.ts

@@ -46,8 +46,8 @@ export async function readMongoImg({ id }: { id: string }) {
   return data?.binary;
 }
 
-export async function delImgByFileIdList(fileIds: string[]) {
+export async function delImgByRelatedId(relateIds: string[]) {
   return MongoImage.deleteMany({
-    'metadata.fileId': { $in: fileIds.map((item) => String(item)) }
+    'metadata.relatedId': { $in: relateIds.map((id) => String(id)) }
   });
 }

packages/service/common/file/image/schema.ts

@@ -35,6 +35,8 @@ try {
   ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
   ImageSchema.index({ type: 1 });
   ImageSchema.index({ teamId: 1 });
+  ImageSchema.index({ createTime: 1 });
+  ImageSchema.index({ 'metadata.relatedId': 1 });
 } catch (error) {
   console.log(error);
 }

packages/service/common/file/load/pdf.ts

@@ -1,68 +0,0 @@
-import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs';
-// @ts-ignore
-import('pdfjs-dist/legacy/build/pdf.worker.min.mjs');
-import { ReadFileParams } from './type';
-
-type TokenType = {
-  str: string;
-  dir: string;
-  width: number;
-  height: number;
-  transform: number[];
-  fontName: string;
-  hasEOL: boolean;
-};
-
-export const readPdfFile = async ({ path }: ReadFileParams) => {
-  const readPDFPage = async (doc: any, pageNo: number) => {
-    const page = await doc.getPage(pageNo);
-    const tokenizedText = await page.getTextContent();
-
-    const viewport = page.getViewport({ scale: 1 });
-    const pageHeight = viewport.height;
-    const headerThreshold = pageHeight * 0.95;
-    const footerThreshold = pageHeight * 0.05;
-
-    const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => {
-      return (
-        !token.transform ||
-        (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold)
-      );
-    });
-
-    // concat empty string 'hasEOL'
-    for (let i = 0; i < pageTexts.length; i++) {
-      const item = pageTexts[i];
-      if (item.str === '' && pageTexts[i - 1]) {
-        pageTexts[i - 1].hasEOL = item.hasEOL;
-        pageTexts.splice(i, 1);
-        i--;
-      }
-    }
-
-    page.cleanup();
-
-    return pageTexts
-      .map((token) => {
-        const paragraphEnd = token.hasEOL && /([。？！.?!\n\r]|(\r\n))$/.test(token.str);
-
-        return paragraphEnd ? `${token.str}\n` : token.str;
-      })
-      .join('');
-  };
-
-  const loadingTask = pdfjs.getDocument(path);
-  const doc = await loadingTask.promise;
-
-  const pageTextPromises = [];
-  for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
-    pageTextPromises.push(readPDFPage(doc, pageNo));
-  }
-  const pageTexts = await Promise.all(pageTextPromises);
-
-  loadingTask.destroy();
-
-  return {
-    rawText: pageTexts.join('')
-  };
-};

packages/service/common/file/load/type.d.ts

@@ -1,18 +0,0 @@
-export type ReadFileParams = {
-  preview: boolean;
-  teamId: string;
-  path: string;
-  metadata?: Record<string, any>;
-};
-
-export type ReadFileResponse = {
-  rawText: string;
-};
-
-export type ReadFileBufferItemType = ReadFileParams & {
-  rawText: string;
-};
-
-declare global {
-  var readFileBuffers: ReadFileBufferItemType[];
-}

packages/service/common/file/load/utils.ts

@@ -1,50 +0,0 @@
-import { readPdfFile } from './pdf';
-import { readDocFle } from './word';
-import { ReadFileBufferItemType, ReadFileParams } from './type';
-
-global.readFileBuffers = global.readFileBuffers || [];
-
-const bufferMaxSize = 200;
-
-export const pushFileReadBuffer = (params: ReadFileBufferItemType) => {
-  global.readFileBuffers.push(params);
-
-  if (global.readFileBuffers.length > bufferMaxSize) {
-    global.readFileBuffers.shift();
-  }
-};
-export const getReadFileBuffer = ({ path, teamId }: ReadFileParams) =>
-  global.readFileBuffers.find((item) => item.path === path && item.teamId === teamId);
-
-export const readFileContent = async (params: ReadFileParams) => {
-  const { path } = params;
-
-  const buffer = getReadFileBuffer(params);
-
-  if (buffer) {
-    return buffer;
-  }
-
-  const extension = path?.split('.')?.pop()?.toLowerCase() || '';
-
-  const { rawText } = await (async () => {
-    switch (extension) {
-      case 'pdf':
-        return readPdfFile(params);
-      case 'docx':
-        return readDocFle(params);
-      default:
-        return Promise.reject('Only support .pdf, .docx');
-    }
-  })();
-
-  pushFileReadBuffer({
-    ...params,
-    rawText
-  });
-
-  return {
-    ...params,
-    rawText
-  };
-};

packages/service/common/file/load/word.ts

@@ -1,22 +0,0 @@
-import mammoth from 'mammoth';
-import { htmlToMarkdown } from '../../string/markdown';
-import { ReadFileParams } from './type';
-/**
- * read docx to markdown
- */
-export const readDocFle = async ({ path, metadata = {} }: ReadFileParams) => {
-  try {
-    const { value: html } = await mammoth.convertToHtml({
-      path
-    });
-
-    const md = await htmlToMarkdown(html);
-
-    return {
-      rawText: md
-    };
-  } catch (error) {
-    console.log('error doc read:', error);
-    return Promise.reject('Can not read doc file, please convert to PDF');
-  }
-};

packages/service/common/file/multer.ts

@@ -3,7 +3,6 @@ import multer from 'multer';
 import path from 'path';
 import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants';
 import { getNanoid } from '@fastgpt/global/common/string/tools';
-import { tmpFileDirPath } from './constants';
 
 type FileType = {
   fieldname: string;
@@ -15,8 +14,6 @@ type FileType = {
   size: number;
 };
 
-const expiredTime = 30 * 60 * 1000;
-
 export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
   maxSize *= 1024 * 1024;
   class UploadModel {
@@ -31,15 +28,16 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
         // },
         filename: async (req, file, cb) => {
           const { ext } = path.parse(decodeURIComponent(file.originalname));
-          cb(null, `${Date.now() + expiredTime}-${getNanoid(32)}${ext}`);
+          cb(null, `${getNanoid(32)}${ext}`);
         }
       })
-    }).any();
+    }).single('file');
 
     async doUpload<T = Record<string, any>>(req: NextApiRequest, res: NextApiResponse) {
       return new Promise<{
-        files: FileType[];
-        metadata: T;
+        file: FileType;
+        metadata: Record<string, any>;
+        data: T;
         bucketName?: `${BucketNameEnum}`;
       }>((resolve, reject) => {
         // @ts-ignore
@@ -54,20 +52,28 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
             return reject('BucketName is invalid');
           }
 
+          // @ts-ignore
+          const file = req.file as FileType;
+
           resolve({
-            ...req.body,
-            files:
-              // @ts-ignore
-              req.files?.map((file) => ({
-                ...file,
-                originalname: decodeURIComponent(file.originalname)
-              })) || [],
+            file: {
+              ...file,
+              originalname: decodeURIComponent(file.originalname)
+            },
+            bucketName,
             metadata: (() => {
               if (!req.body?.metadata) return {};
               try {
                 return JSON.parse(req.body.metadata);
               } catch (error) {
-                console.log(error);
+                return {};
+              }
+            })(),
+            data: (() => {
+              if (!req.body?.data) return {};
+              try {
+                return JSON.parse(req.body.data);
+              } catch (error) {
                 return {};
               }
             })()

packages/service/common/file/utils.ts

@@ -1,5 +1,4 @@
 import fs from 'fs';
-import { tmpFileDirPath } from './constants';
 
 export const removeFilesByPaths = (paths: string[]) => {
   paths.forEach((path) => {
@@ -10,24 +9,3 @@ export const removeFilesByPaths = (paths: string[]) => {
     });
   });
 };
-
-/* cron job.  check expired tmp files */
-export const checkExpiredTmpFiles = () => {
-  // get all file name
-  const files = fs.readdirSync(tmpFileDirPath).map((name) => {
-    const timestampStr = name.split('-')[0];
-    const expiredTimestamp = timestampStr ? Number(timestampStr) : 0;
-
-    return {
-      filename: name,
-      expiredTimestamp,
-      path: `${tmpFileDirPath}/${name}`
-    };
-  });
-
-  // count expiredFiles
-  const expiredFiles = files.filter((item) => item.expiredTimestamp < Date.now());
-
-  // remove expiredFiles
-  removeFilesByPaths(expiredFiles.map((item) => item.path));
-};