4.6.7-alpha commit (#743)

Co-authored-by: Archer <545436317@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>

packages/service/common/file/gridfs/controller.ts

@@ -3,9 +3,10 @@ import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
 import fsp from 'fs/promises';
 import fs from 'fs';
 import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
-import { delImgByFileIdList } from '../image/controller';
+import { MongoFileSchema } from './schema';
 
 export function getGFSCollection(bucket: `${BucketNameEnum}`) {
+  MongoFileSchema;
   return connectionMongo.connection.db.collection(`${bucket}.files`);
 }
 export function getGridBucket(bucket: `${BucketNameEnum}`) {
@@ -21,6 +22,7 @@ export async function uploadFile({
   tmbId,
   path,
   filename,
+  contentType,
   metadata = {}
 }: {
   bucketName: `${BucketNameEnum}`;
@@ -28,6 +30,7 @@ export async function uploadFile({
   tmbId: string;
   path: string;
   filename: string;
+  contentType?: string;
   metadata?: Record<string, any>;
 }) {
   if (!path) return Promise.reject(`filePath is empty`);
@@ -44,7 +47,7 @@ export async function uploadFile({
 
   const stream = bucket.openUploadStream(filename, {
     metadata,
-    contentType: metadata?.contentType
+    contentType
   });
 
   // save to gridfs
@@ -96,40 +99,6 @@ export async function delFileByFileIdList({
     }
   }
 }
-// delete file by metadata(datasetId)
-export async function delFileByMetadata({
-  bucketName,
-  datasetId
-}: {
-  bucketName: `${BucketNameEnum}`;
-  datasetId?: string;
-}) {
-  const bucket = getGridBucket(bucketName);
-
-  const files = await bucket
-    .find(
-      {
-        ...(datasetId && { 'metadata.datasetId': datasetId })
-      },
-      {
-        projection: {
-          _id: 1
-        }
-      }
-    )
-    .toArray();
-
-  const idList = files.map((item) => String(item._id));
-
-  // delete img
-  await delImgByFileIdList(idList);
-
-  // delete file
-  await delFileByFileIdList({
-    bucketName,
-    fileIdList: idList
-  });
-}
 
 export async function getDownloadStream({
   bucketName,

packages/service/common/file/gridfs/schema.ts

@@ -0,0 +1,15 @@
+import { connectionMongo, type Model } from '../../mongo';
+const { Schema, model, models } = connectionMongo;
+
+const FileSchema = new Schema({});
+
+try {
+  FileSchema.index({ 'metadata.teamId': 1 });
+  FileSchema.index({ 'metadata.uploadDate': -1 });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoFileSchema = models['dataset.files'] || model('dataset.files', FileSchema);
+
+MongoFileSchema.syncIndexes();

packages/service/common/file/image/controller.ts

@@ -46,8 +46,8 @@ export async function readMongoImg({ id }: { id: string }) {
   return data?.binary;
 }
 
-export async function delImgByFileIdList(fileIds: string[]) {
+export async function delImgByRelatedId(relateIds: string[]) {
   return MongoImage.deleteMany({
-    'metadata.fileId': { $in: fileIds.map((item) => String(item)) }
+    'metadata.relatedId': { $in: relateIds.map((id) => String(id)) }
   });
 }

packages/service/common/file/image/schema.ts

@@ -35,6 +35,8 @@ try {
   ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 });
   ImageSchema.index({ type: 1 });
   ImageSchema.index({ teamId: 1 });
+  ImageSchema.index({ createTime: 1 });
+  ImageSchema.index({ 'metadata.relatedId': 1 });
 } catch (error) {
   console.log(error);
 }

packages/service/common/file/load/pdf.ts

@@ -1,68 +0,0 @@
-import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs';
-// @ts-ignore
-import('pdfjs-dist/legacy/build/pdf.worker.min.mjs');
-import { ReadFileParams } from './type';
-
-type TokenType = {
-  str: string;
-  dir: string;
-  width: number;
-  height: number;
-  transform: number[];
-  fontName: string;
-  hasEOL: boolean;
-};
-
-export const readPdfFile = async ({ path }: ReadFileParams) => {
-  const readPDFPage = async (doc: any, pageNo: number) => {
-    const page = await doc.getPage(pageNo);
-    const tokenizedText = await page.getTextContent();
-
-    const viewport = page.getViewport({ scale: 1 });
-    const pageHeight = viewport.height;
-    const headerThreshold = pageHeight * 0.95;
-    const footerThreshold = pageHeight * 0.05;
-
-    const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => {
-      return (
-        !token.transform ||
-        (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold)
-      );
-    });
-
-    // concat empty string 'hasEOL'
-    for (let i = 0; i < pageTexts.length; i++) {
-      const item = pageTexts[i];
-      if (item.str === '' && pageTexts[i - 1]) {
-        pageTexts[i - 1].hasEOL = item.hasEOL;
-        pageTexts.splice(i, 1);
-        i--;
-      }
-    }
-
-    page.cleanup();
-
-    return pageTexts
-      .map((token) => {
-        const paragraphEnd = token.hasEOL && /([。？！.?!\n\r]|(\r\n))$/.test(token.str);
-
-        return paragraphEnd ? `${token.str}\n` : token.str;
-      })
-      .join('');
-  };
-
-  const loadingTask = pdfjs.getDocument(path);
-  const doc = await loadingTask.promise;
-
-  const pageTextPromises = [];
-  for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) {
-    pageTextPromises.push(readPDFPage(doc, pageNo));
-  }
-  const pageTexts = await Promise.all(pageTextPromises);
-
-  loadingTask.destroy();
-
-  return {
-    rawText: pageTexts.join('')
-  };
-};

packages/service/common/file/load/type.d.ts

@@ -1,18 +0,0 @@
-export type ReadFileParams = {
-  preview: boolean;
-  teamId: string;
-  path: string;
-  metadata?: Record<string, any>;
-};
-
-export type ReadFileResponse = {
-  rawText: string;
-};
-
-export type ReadFileBufferItemType = ReadFileParams & {
-  rawText: string;
-};
-
-declare global {
-  var readFileBuffers: ReadFileBufferItemType[];
-}

packages/service/common/file/load/utils.ts

@@ -1,50 +0,0 @@
-import { readPdfFile } from './pdf';
-import { readDocFle } from './word';
-import { ReadFileBufferItemType, ReadFileParams } from './type';
-
-global.readFileBuffers = global.readFileBuffers || [];
-
-const bufferMaxSize = 200;
-
-export const pushFileReadBuffer = (params: ReadFileBufferItemType) => {
-  global.readFileBuffers.push(params);
-
-  if (global.readFileBuffers.length > bufferMaxSize) {
-    global.readFileBuffers.shift();
-  }
-};
-export const getReadFileBuffer = ({ path, teamId }: ReadFileParams) =>
-  global.readFileBuffers.find((item) => item.path === path && item.teamId === teamId);
-
-export const readFileContent = async (params: ReadFileParams) => {
-  const { path } = params;
-
-  const buffer = getReadFileBuffer(params);
-
-  if (buffer) {
-    return buffer;
-  }
-
-  const extension = path?.split('.')?.pop()?.toLowerCase() || '';
-
-  const { rawText } = await (async () => {
-    switch (extension) {
-      case 'pdf':
-        return readPdfFile(params);
-      case 'docx':
-        return readDocFle(params);
-      default:
-        return Promise.reject('Only support .pdf, .docx');
-    }
-  })();
-
-  pushFileReadBuffer({
-    ...params,
-    rawText
-  });
-
-  return {
-    ...params,
-    rawText
-  };
-};

packages/service/common/file/load/word.ts

@@ -1,22 +0,0 @@
-import mammoth from 'mammoth';
-import { htmlToMarkdown } from '../../string/markdown';
-import { ReadFileParams } from './type';
-/**
- * read docx to markdown
- */
-export const readDocFle = async ({ path, metadata = {} }: ReadFileParams) => {
-  try {
-    const { value: html } = await mammoth.convertToHtml({
-      path
-    });
-
-    const md = await htmlToMarkdown(html);
-
-    return {
-      rawText: md
-    };
-  } catch (error) {
-    console.log('error doc read:', error);
-    return Promise.reject('Can not read doc file, please convert to PDF');
-  }
-};

packages/service/common/file/multer.ts

@@ -3,7 +3,6 @@ import multer from 'multer';
 import path from 'path';
 import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants';
 import { getNanoid } from '@fastgpt/global/common/string/tools';
-import { tmpFileDirPath } from './constants';
 
 type FileType = {
   fieldname: string;
@@ -15,8 +14,6 @@ type FileType = {
   size: number;
 };
 
-const expiredTime = 30 * 60 * 1000;
-
 export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
   maxSize *= 1024 * 1024;
   class UploadModel {
@@ -31,15 +28,16 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
         // },
         filename: async (req, file, cb) => {
           const { ext } = path.parse(decodeURIComponent(file.originalname));
-          cb(null, `${Date.now() + expiredTime}-${getNanoid(32)}${ext}`);
+          cb(null, `${getNanoid(32)}${ext}`);
         }
       })
-    }).any();
+    }).single('file');
 
     async doUpload<T = Record<string, any>>(req: NextApiRequest, res: NextApiResponse) {
       return new Promise<{
-        files: FileType[];
-        metadata: T;
+        file: FileType;
+        metadata: Record<string, any>;
+        data: T;
         bucketName?: `${BucketNameEnum}`;
       }>((resolve, reject) => {
         // @ts-ignore
@@ -54,20 +52,28 @@ export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => {
             return reject('BucketName is invalid');
           }
 
+          // @ts-ignore
+          const file = req.file as FileType;
+
           resolve({
-            ...req.body,
-            files:
-              // @ts-ignore
-              req.files?.map((file) => ({
-                ...file,
-                originalname: decodeURIComponent(file.originalname)
-              })) || [],
+            file: {
+              ...file,
+              originalname: decodeURIComponent(file.originalname)
+            },
+            bucketName,
             metadata: (() => {
               if (!req.body?.metadata) return {};
               try {
                 return JSON.parse(req.body.metadata);
               } catch (error) {
-                console.log(error);
+                return {};
+              }
+            })(),
+            data: (() => {
+              if (!req.body?.data) return {};
+              try {
+                return JSON.parse(req.body.data);
+              } catch (error) {
                 return {};
               }
             })()

packages/service/common/file/utils.ts

@@ -1,5 +1,4 @@
 import fs from 'fs';
-import { tmpFileDirPath } from './constants';
 
 export const removeFilesByPaths = (paths: string[]) => {
   paths.forEach((path) => {
@@ -10,24 +9,3 @@ export const removeFilesByPaths = (paths: string[]) => {
     });
   });
 };
-
-/* cron job.  check expired tmp files */
-export const checkExpiredTmpFiles = () => {
-  // get all file name
-  const files = fs.readdirSync(tmpFileDirPath).map((name) => {
-    const timestampStr = name.split('-')[0];
-    const expiredTimestamp = timestampStr ? Number(timestampStr) : 0;
-
-    return {
-      filename: name,
-      expiredTimestamp,
-      path: `${tmpFileDirPath}/${name}`
-    };
-  });
-
-  // count expiredFiles
-  const expiredFiles = files.filter((item) => item.expiredTimestamp < Date.now());
-
-  // remove expiredFiles
-  removeFilesByPaths(expiredFiles.map((item) => item.path));
-};

packages/service/common/string/cheerio.ts

@@ -64,41 +64,39 @@ export const urlsFetch = async ({
 }: UrlFetchParams): Promise<UrlFetchResponse> => {
   urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));
 
-  const response = (
-    await Promise.all(
-      urlList.map(async (url) => {
-        try {
-          const fetchRes = await axios.get(url, {
-            timeout: 30000
-          });
+  const response = await Promise.all(
+    urlList.map(async (url) => {
+      try {
+        const fetchRes = await axios.get(url, {
+          timeout: 30000
+        });
 
-          const $ = cheerio.load(fetchRes.data);
-          const { title, html, usedSelector } = cheerioToHtml({
-            fetchUrl: url,
-            $,
-            selector
-          });
-          const md = await htmlToMarkdown(html);
+        const $ = cheerio.load(fetchRes.data);
+        const { title, html, usedSelector } = cheerioToHtml({
+          fetchUrl: url,
+          $,
+          selector
+        });
+        const md = await htmlToMarkdown(html);
 
-          return {
-            url,
-            title,
-            content: md,
-            selector: usedSelector
-          };
-        } catch (error) {
-          console.log(error, 'fetch error');
+        return {
+          url,
+          title,
+          content: md,
+          selector: usedSelector
+        };
+      } catch (error) {
+        console.log(error, 'fetch error');
 
-          return {
-            url,
-            title: '',
-            content: '',
-            selector: ''
-          };
-        }
-      })
-    )
-  ).filter((item) => item.content);
+        return {
+          url,
+          title: '',
+          content: '',
+          selector: ''
+        };
+      }
+    })
+  );
 
   return response;
 };

packages/service/common/vectorStore/controller.d.ts

@@ -1,21 +1,19 @@
 export type DeleteDatasetVectorProps = {
+  teamId: string;
+
   id?: string;
   datasetIds?: string[];
   collectionIds?: string[];
-
-  collectionId?: string;
-  dataIds?: string[];
+  idList?: string[];
 };
 
 export type InsertVectorProps = {
   teamId: string;
-  tmbId: string;
   datasetId: string;
   collectionId: string;
-  dataId: string;
 };
 
 export type EmbeddingRecallProps = {
-  similarity?: number;
   datasetIds: string[];
+  similarity?: number;
 };

packages/service/common/vectorStore/controller.ts

@@ -10,6 +10,7 @@ const getVectorObj = () => {
 export const initVectorStore = getVectorObj().init;
 export const deleteDatasetDataVector = getVectorObj().delete;
 export const recallFromVectorStore = getVectorObj().recall;
+export const checkVectorDataExist = getVectorObj().checkDataExist;
 export const getVectorDataByTime = getVectorObj().getVectorDataByTime;
 export const getVectorCountByTeamId = getVectorObj().getVectorCountByTeamId;
 
@@ -21,7 +22,7 @@ export const insertDatasetDataVector = async ({
   query: string;
   model: string;
 }) => {
-  const { vectors, tokens } = await getVectorsByText({
+  const { vectors, charsLength } = await getVectorsByText({
     model,
     input: query
   });
@@ -31,32 +32,27 @@ export const insertDatasetDataVector = async ({
   });
 
   return {
-    tokens,
+    charsLength,
     insertId
   };
 };
 
 export const updateDatasetDataVector = async ({
   id,
-  query,
-  model
-}: {
+  ...props
+}: InsertVectorProps & {
   id: string;
   query: string;
   model: string;
 }) => {
-  // get vector
-  const { vectors, tokens } = await getVectorsByText({
-    model,
-    input: query
+  // insert new vector
+  const { charsLength, insertId } = await insertDatasetDataVector(props);
+
+  // delete old vector
+  await deleteDatasetDataVector({
+    teamId: props.teamId,
+    id
   });
 
-  await getVectorObj().update({
-    id,
-    vectors
-  });
-
-  return {
-    tokens
-  };
+  return { charsLength, insertId };
 };

packages/service/common/vectorStore/pg/class.ts

@@ -1,20 +1,20 @@
 import {
   initPg,
   insertDatasetDataVector,
-  updateDatasetDataVector,
   deleteDatasetDataVector,
   embeddingRecall,
   getVectorDataByTime,
-  getVectorCountByTeamId
+  getVectorCountByTeamId,
+  checkDataExist
 } from './controller';
 
 export class PgVector {
   constructor() {}
   init = initPg;
   insert = insertDatasetDataVector;
-  update = updateDatasetDataVector;
   delete = deleteDatasetDataVector;
   recall = embeddingRecall;
+  checkDataExist = checkDataExist;
   getVectorCountByTeamId = getVectorCountByTeamId;
   getVectorDataByTime = getVectorDataByTime;
 }

packages/service/common/vectorStore/pg/controller.ts

@@ -4,7 +4,7 @@ import { delay } from '@fastgpt/global/common/system/utils';
 import { PgClient, connectPg } from './index';
 import { PgSearchRawType } from '@fastgpt/global/core/dataset/api';
 import { EmbeddingRecallItemType } from '../type';
-import { DeleteDatasetVectorProps, EmbeddingRecallProps } from '../controller.d';
+import { DeleteDatasetVectorProps, EmbeddingRecallProps, InsertVectorProps } from '../controller.d';
 import dayjs from 'dayjs';
 
 export async function initPg() {
@@ -16,11 +16,9 @@ export async function initPg() {
           id BIGSERIAL PRIMARY KEY,
           vector VECTOR(1536) NOT NULL,
           team_id VARCHAR(50) NOT NULL,
-          tmb_id VARCHAR(50) NOT NULL,
           dataset_id VARCHAR(50) NOT NULL,
           collection_id VARCHAR(50) NOT NULL,
-          data_id VARCHAR(50) NOT NULL,
-          createTime TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+          createtime TIMESTAMP DEFAULT CURRENT_TIMESTAMP
       );
     `);
 
@@ -34,26 +32,21 @@ export async function initPg() {
   }
 }
 
-export const insertDatasetDataVector = async (props: {
-  teamId: string;
-  tmbId: string;
-  datasetId: string;
-  collectionId: string;
-  dataId: string;
-  vectors: number[][];
-  retry?: number;
-}): Promise<{ insertId: string }> => {
-  const { dataId, teamId, tmbId, datasetId, collectionId, vectors, retry = 3 } = props;
+export const insertDatasetDataVector = async (
+  props: InsertVectorProps & {
+    vectors: number[][];
+    retry?: number;
+  }
+): Promise<{ insertId: string }> => {
+  const { teamId, datasetId, collectionId, vectors, retry = 3 } = props;
   try {
     const { rows } = await PgClient.insert(PgDatasetTableName, {
       values: [
         [
           { key: 'vector', value: `[${vectors[0]}]` },
           { key: 'team_id', value: String(teamId) },
-          { key: 'tmb_id', value: String(tmbId) },
           { key: 'dataset_id', value: datasetId },
-          { key: 'collection_id', value: collectionId },
-          { key: 'data_id', value: String(dataId) }
+          { key: 'collection_id', value: collectionId }
         ]
       ]
     });
@@ -72,48 +65,33 @@ export const insertDatasetDataVector = async (props: {
   }
 };
 
-export const updateDatasetDataVector = async (props: {
-  id: string;
-  vectors: number[][];
-  retry?: number;
-}): Promise<void> => {
-  const { id, vectors, retry = 2 } = props;
-  try {
-    // update pg
-    await PgClient.update(PgDatasetTableName, {
-      where: [['id', id]],
-      values: [{ key: 'vector', value: `[${vectors[0]}]` }]
-    });
-  } catch (error) {
-    if (retry <= 0) {
-      return Promise.reject(error);
-    }
-    await delay(500);
-    return updateDatasetDataVector({
-      ...props,
-      retry: retry - 1
-    });
-  }
-};
-
 export const deleteDatasetDataVector = async (
   props: DeleteDatasetVectorProps & {
     retry?: number;
   }
 ): Promise<any> => {
-  const { id, datasetIds, collectionIds, collectionId, dataIds, retry = 2 } = props;
+  const { teamId, id, datasetIds, collectionIds, idList, retry = 2 } = props;
+
+  const teamIdWhere = `team_id='${String(teamId)}' AND`;
 
   const where = await (() => {
-    if (id) return `id=${id}`;
-    if (datasetIds) return `dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})`;
-    if (collectionIds) {
-      return `collection_id IN (${collectionIds.map((id) => `'${String(id)}'`).join(',')})`;
-    }
-    if (collectionId && dataIds) {
-      return `collection_id='${String(collectionId)}' and data_id IN (${dataIds
+    if (id) return `${teamIdWhere} id=${id}`;
+
+    if (datasetIds) {
+      return `${teamIdWhere} dataset_id IN (${datasetIds
         .map((id) => `'${String(id)}'`)
         .join(',')})`;
     }
+
+    if (collectionIds) {
+      return `${teamIdWhere} collection_id IN (${collectionIds
+        .map((id) => `'${String(id)}'`)
+        .join(',')})`;
+    }
+
+    if (idList) {
+      return `${teamIdWhere} id IN (${idList.map((id) => `'${String(id)}'`).join(',')})`;
+    }
     return Promise.reject('deleteDatasetData: no where');
   })();
 
@@ -142,13 +120,13 @@ export const embeddingRecall = async (
 ): Promise<{
   results: EmbeddingRecallItemType[];
 }> => {
-  const { vectors, limit, similarity = 0, datasetIds, retry = 2 } = props;
+  const { datasetIds, vectors, limit, similarity = 0, retry = 2 } = props;
 
   try {
     const results: any = await PgClient.query(
       `BEGIN;
         SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 100};
-        select id, collection_id, data_id, (vector <#> '[${vectors[0]}]') * -1 AS score 
+        select id, collection_id, (vector <#> '[${vectors[0]}]') * -1 AS score 
           from ${PgDatasetTableName} 
           where dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})
               AND vector <#> '[${vectors[0]}]' < -${similarity}
@@ -158,21 +136,10 @@ export const embeddingRecall = async (
 
     const rows = results?.[2]?.rows as PgSearchRawType[];
 
-    // concat same data_id
-    const filterRows: PgSearchRawType[] = [];
-    let set = new Set<string>();
-    for (const row of rows) {
-      if (!set.has(row.data_id)) {
-        filterRows.push(row);
-        set.add(row.data_id);
-      }
-    }
-
     return {
-      results: filterRows.map((item) => ({
+      results: rows.map((item) => ({
         id: item.id,
         collectionId: item.collection_id,
-        dataId: item.data_id,
         score: item.score
       }))
     };
@@ -184,7 +151,11 @@ export const embeddingRecall = async (
   }
 };
 
-// bill
+export const checkDataExist = async (id: string) => {
+  const { rows } = await PgClient.query(`SELECT id FROM ${PgDatasetTableName} WHERE id=${id};`);
+
+  return rows.length > 0;
+};
 export const getVectorCountByTeamId = async (teamId: string) => {
   const total = await PgClient.count(PgDatasetTableName, {
     where: [['team_id', String(teamId)]]
@@ -193,15 +164,20 @@ export const getVectorCountByTeamId = async (teamId: string) => {
   return total;
 };
 export const getVectorDataByTime = async (start: Date, end: Date) => {
-  const { rows } = await PgClient.query<{ id: string; data_id: string }>(`SELECT id, data_id
+  const { rows } = await PgClient.query<{
+    id: string;
+    team_id: string;
+    dataset_id: string;
+  }>(`SELECT id, team_id, dataset_id
   FROM ${PgDatasetTableName}
-  WHERE createTime BETWEEN '${dayjs(start).format('YYYY-MM-DD')}' AND '${dayjs(end).format(
-    'YYYY-MM-DD 23:59:59'
+  WHERE createtime BETWEEN '${dayjs(start).format('YYYY-MM-DD HH:mm:ss')}' AND '${dayjs(end).format(
+    'YYYY-MM-DD HH:mm:ss'
   )}';
   `);
 
   return rows.map((item) => ({
     id: item.id,
-    dataId: item.data_id
+    datasetId: item.dataset_id,
+    teamId: item.team_id
   }));
 };

packages/service/common/vectorStore/type.d.ts

@@ -7,6 +7,5 @@ declare global {
 export type EmbeddingRecallItemType = {
   id: string;
   collectionId: string;
-  dataId: string;
   score: number;
 };