feat: gridfs save file

client/src/service/errorCode.ts

@@ -39,7 +39,8 @@ export enum ERROR_ENUM {
   unAuthorization = 'unAuthorization',
   insufficientQuota = 'insufficientQuota',
   unAuthModel = 'unAuthModel',
-  unAuthKb = 'unAuthKb'
+  unAuthKb = 'unAuthKb',
+  unAuthFile = 'unAuthFile'
 }
 export const ERROR_RESPONSE: Record<
   any,
@@ -73,5 +74,11 @@ export const ERROR_RESPONSE: Record<
     statusText: ERROR_ENUM.unAuthKb,
     message: '无权使用该知识库',
     data: null
+  },
+  [ERROR_ENUM.unAuthFile]: {
+    code: 513,
+    statusText: ERROR_ENUM.unAuthFile,
+    message: '无权阅读该文件',
+    data: null
   }
 };

client/src/service/events/generateQA.ts

@@ -5,7 +5,7 @@ import { TrainingModeEnum } from '@/constants/plugin';
 import { ERROR_ENUM } from '../errorCode';
 import { sendInform } from '@/pages/api/user/inform/send';
 import { authBalanceByUid } from '../utils/auth';
-import { axiosConfig, getAIChatApi } from '../ai/openai';
+import { axiosConfig, getAIChatApi } from '../lib/openai';
 import { ChatCompletionRequestMessage } from 'openai';
 import { modelToolMap } from '@/utils/plugin';
 import { gptMessage2ChatType } from '@/utils/adapt';

client/src/service/lib/gridfs.ts

@@ -0,0 +1,119 @@
+import mongoose, { Types } from 'mongoose';
+import fs from 'fs';
+import fsp from 'fs/promises';
+import { ERROR_ENUM } from '../errorCode';
+
+enum BucketNameEnum {
+  dataset = 'dataset'
+}
+
+type FileInfo = {
+  id: string;
+  filename: string;
+  size: number;
+  contentType: string;
+  encoding: string;
+};
+
+export class GridFSStorage {
+  readonly type = 'gridfs';
+  readonly bucket: `${BucketNameEnum}`;
+  readonly uid: string;
+
+  constructor(bucket: `${BucketNameEnum}`, uid: string) {
+    this.bucket = bucket;
+    this.uid = String(uid);
+  }
+  GridFSBucket() {
+    return new mongoose.mongo.GridFSBucket(mongoose.connection.db, {
+      bucketName: this.bucket
+    });
+  }
+
+  async save({
+    path,
+    filename,
+    metadata = {}
+  }: {
+    path: string;
+    filename: string;
+    metadata?: Record<string, any>;
+  }) {
+    if (!path) return Promise.reject(`filePath is empty`);
+    if (!filename) return Promise.reject(`filename is empty`);
+
+    const stats = await fsp.stat(path);
+    if (!stats.isFile()) return Promise.reject(`${path} is not a file`);
+
+    metadata.userId = this.uid;
+    // create a gridfs bucket
+    const bucket = this.GridFSBucket();
+
+    const stream = bucket.openUploadStream(filename, {
+      metadata,
+      contentType: metadata?.contentType
+    });
+
+    // save to gridfs
+    await new Promise((resolve, reject) => {
+      fs.createReadStream(path)
+        .pipe(stream as any)
+        .on('finish', resolve)
+        .on('error', reject);
+    });
+
+    return String(stream.id);
+  }
+  async findAndAuthFile(id: string): Promise<FileInfo> {
+    if (!id) {
+      return Promise.reject(`id is empty`);
+    }
+
+    // create a gridfs bucket
+    const bucket = this.GridFSBucket();
+
+    // check if file exists
+    const files = await bucket.find({ _id: new Types.ObjectId(id) }).toArray();
+    const file = files.shift();
+    if (!file) {
+      return Promise.reject(`file not found`);
+    }
+
+    if (String(file.metadata?.userId) !== this.uid) {
+      return Promise.reject(ERROR_ENUM.unAuthFile);
+    }
+
+    return {
+      id: String(file._id),
+      filename: file.filename,
+      contentType: file.metadata?.contentType,
+      encoding: file.metadata?.encoding,
+      size: file.length
+    };
+  }
+
+  async delete(id: string) {
+    await this.findAndAuthFile(id);
+    const bucket = this.GridFSBucket();
+
+    await bucket.delete(new Types.ObjectId(id));
+    return true;
+  }
+
+  async download(id: string) {
+    await this.findAndAuthFile(id);
+
+    const bucket = this.GridFSBucket();
+
+    const stream = bucket.openDownloadStream(new Types.ObjectId(id));
+
+    const buf: Buffer = await new Promise((resolve, reject) => {
+      const buffers: Buffer[] = [];
+      stream.on('data', (data) => buffers.push(data));
+      stream.on('error', reject);
+      stream.on('end', () => resolve(Buffer.concat(buffers)));
+    });
+
+    return buf;
+  }
+}

client/src/service/moduleDispatch/agent/classifyQuestion.ts

@@ -2,7 +2,7 @@ import { adaptChatItem_openAI } from '@/utils/plugin/openai';
 import { ChatContextFilter } from '@/service/utils/chat/index';
 import type { ChatHistoryItemResType, ChatItemType } from '@/types/chat';
 import { ChatModuleEnum, ChatRoleEnum, TaskResponseKeyEnum } from '@/constants/chat';
-import { getAIChatApi, axiosConfig } from '@/service/ai/openai';
+import { getAIChatApi, axiosConfig } from '@/service/lib/openai';
 import type { ClassifyQuestionAgentItemType } from '@/types/app';
 import { countModelPrice } from '@/service/events/pushBill';
 import { UserModelSchema } from '@/types/mongoSchema';

client/src/service/moduleDispatch/agent/extract.ts

@@ -2,7 +2,7 @@ import { adaptChatItem_openAI } from '@/utils/plugin/openai';
 import { ChatContextFilter } from '@/service/utils/chat/index';
 import type { ChatHistoryItemResType, ChatItemType } from '@/types/chat';
 import { ChatModuleEnum, ChatRoleEnum, TaskResponseKeyEnum } from '@/constants/chat';
-import { getAIChatApi, axiosConfig } from '@/service/ai/openai';
+import { getAIChatApi, axiosConfig } from '@/service/lib/openai';
 import type { ContextExtractAgentItemType } from '@/types/app';
 import { ContextExtractEnum } from '@/constants/flow/flowField';
 import { countModelPrice } from '@/service/events/pushBill';

client/src/service/moduleDispatch/chat/oneapi.ts

@@ -8,7 +8,7 @@ import type { ChatHistoryItemResType } from '@/types/chat';
 import { ChatModuleEnum, ChatRoleEnum, sseResponseEventEnum } from '@/constants/chat';
 import { SSEParseData, parseStreamChunk } from '@/utils/sse';
 import { textAdaptGptResponse } from '@/utils/adapt';
-import { getAIChatApi, axiosConfig } from '@/service/ai/openai';
+import { getAIChatApi, axiosConfig } from '@/service/lib/openai';
 import { TaskResponseKeyEnum } from '@/constants/chat';
 import { getChatModel } from '@/service/utils/data';
 import { countModelPrice } from '@/service/events/pushBill';

client/src/service/plugins/useSearch.ts

@@ -1,34 +0,0 @@
-import axios from 'axios';
-
-{
-  /*Bing 搜索*/
-}
-const BingSearch = async (wait_val: string) => {
-  const response = await axios.post('newbing中转服务器', {
-    prompt: wait_val
-  });
-  const result = response.data.result;
-  return result;
-};
-
-{
-  /*google 搜索*/
-}
-const GoogleSearch = async (wait_val: string) => {
-  const response = await axios.get('https://www.googleapis.com/customsearch/v1', {
-    params: {
-      key: process.env.GOOGLE_KEY,
-      q: wait_val,
-      cx: process.env.searchEngineId,
-      start: 1,
-      num: 3,
-      dateRestrict: 'm[1]' //搜索结果限定为一个月内
-    }
-  });
-  const results = response.data.items;
-  if (results !== null) {
-    const result = results.map((item: { snippet: string }) => item.snippet).join('\n');
-    return result;
-  }
-};
-export { BingSearch, GoogleSearch };