Dataset folder manager (#274)

* feat: retry send

* perf: qa default value

* feat: dataset folder

* feat: kb folder delete and path

* fix: ts

* perf: script load

* feat: fileCard and dataCard

* feat: search file

* feat: max token

* feat: select dataset

* fix: preview chunk

* perf: source update

* export data limit file_id

* docs

* fix: export limit

client/src/service/events/generateQA.ts

@@ -10,6 +10,7 @@ import { ChatCompletionRequestMessage } from 'openai';
 import { modelToolMap } from '@/utils/plugin';
 import { gptMessage2ChatType } from '@/utils/adapt';
 import { addLog } from '../utils/tools';
+import { splitText2Chunks } from '@/utils/file';
 
 const reduceQueue = () => {
   global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -157,7 +158,7 @@ A2:
       console.log('openai error: 生成QA错误');
       console.log(err.response?.status, err.response?.statusText, err.response?.data);
     } else {
-      console.log('生成QA错误:', err);
+      addLog.error('生成 QA 错误', err);
     }
 
     // message error or openai account error
@@ -212,5 +213,16 @@ function formatSplitText(text: string) {
     }
   }
 
+  // empty result. direct split chunk
+  if (result.length === 0) {
+    const splitRes = splitText2Chunks({ text: text, maxLen: 500 });
+    splitRes.chunks.forEach((item) => {
+      result.push({
+        q: item,
+        a: ''
+      });
+    });
+  }
+
   return result;
 }

client/src/service/events/generateVector.ts

@@ -96,9 +96,7 @@ export async function generateVector(): Promise<any> {
         data: err.response?.data
       });
     } else {
-      addLog.info('openai error: 生成向量错误', {
-        err
-      });
+      addLog.error('openai error: 生成向量错误', err);
     }
 
     // message error or openai account error

client/src/service/lib/gridfs.ts

@@ -2,19 +2,12 @@ import mongoose, { Types } from 'mongoose';
 import fs from 'fs';
 import fsp from 'fs/promises';
 import { ERROR_ENUM } from '../errorCode';
+import type { FileInfo } from '@/types/plugin';
 
 enum BucketNameEnum {
   dataset = 'dataset'
 }
 
-type FileInfo = {
-  id: string;
-  filename: string;
-  size: number;
-  contentType: string;
-  encoding: string;
-};
-
 export class GridFSStorage {
   readonly type = 'gridfs';
   readonly bucket: `${BucketNameEnum}`;
@@ -88,6 +81,7 @@ export class GridFSStorage {
       filename: file.filename,
       contentType: file.metadata?.contentType,
       encoding: file.metadata?.encoding,
+      uploadDate: file.uploadDate,
       size: file.length
     };
   }

client/src/service/models/bill.ts

@@ -55,7 +55,6 @@ const BillSchema = new Schema({
 
 try {
   BillSchema.index({ userId: 1 });
-  // BillSchema.index({ time: -1 });
   BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 });
 } catch (error) {
   console.log(error);

client/src/service/models/chatItem.ts

@@ -48,24 +48,7 @@ const ChatItemSchema = new Schema({
     }
   },
   [TaskResponseKeyEnum.responseData]: {
-    type: [
-      {
-        moduleName: String,
-        price: String,
-        model: String,
-        tokens: Number,
-        question: String,
-        answer: String,
-        temperature: Number,
-        maxToken: Number,
-        quoteList: Array,
-        completeMessages: Array,
-        similarity: Number,
-        limit: Number,
-        cqList: Array,
-        cqResult: String
-      }
-    ],
+    type: Array,
     default: []
   }
 });

client/src/service/models/kb.ts

@@ -1,7 +1,13 @@
 import { Schema, model, models, Model } from 'mongoose';
 import { kbSchema as SchemaType } from '@/types/mongoSchema';
+import { KbTypeMap } from '@/constants/kb';
 
 const kbSchema = new Schema({
+  parentId: {
+    type: Schema.Types.ObjectId,
+    ref: 'kb',
+    default: null
+  },
   userId: {
     type: Schema.Types.ObjectId,
     ref: 'user',
@@ -24,6 +30,12 @@ const kbSchema = new Schema({
     required: true,
     default: 'text-embedding-ada-002'
   },
+  type: {
+    type: String,
+    enum: Object.keys(KbTypeMap),
+    required: true,
+    default: 'dataset'
+  },
   tags: {
     type: [String],
     default: []

client/src/service/pg.ts

@@ -13,6 +13,7 @@ export const connectPg = async (): Promise<Pool> => {
     connectionString: process.env.PG_URL,
     max: Number(process.env.DB_MAX_LINK || 5),
     keepAlive: true,
+    idleTimeoutMillis: 30000,
     connectionTimeoutMillis: 5000
   });
 
@@ -107,6 +108,7 @@ class Pg {
       }
       LIMIT ${props.limit || 10} OFFSET ${props.offset || 0}
     `;
+
     const pg = await connectPg();
     return pg.query<T>(sql);
   }

client/src/service/response.ts

@@ -44,7 +44,7 @@ export const jsonRes = <T = any>(
     if (typeof error === 'string') {
       msg = error;
     } else if (proxyError[error?.code]) {
-      msg = '接口连接异常';
+      msg = '网络连接异常';
     } else if (error?.response?.data?.error?.message) {
       msg = error?.response?.data?.error?.message;
     } else if (openaiAccountError[error?.response?.data?.error?.code]) {
@@ -85,7 +85,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => {
   if (typeof error === 'string') {
     msg = error;
   } else if (proxyError[error?.code]) {
-    msg = '接口连接异常';
+    msg = '网络连接异常';
   } else if (error?.response?.data?.error?.message) {
     msg = error?.response?.data?.error?.message;
   } else if (openaiAccountError[error?.response?.data?.error?.code]) {

client/src/service/utils/auth.ts

@@ -155,7 +155,7 @@ export const authUser = async ({
   })();
 
   return {
-    userId: uid,
+    userId: String(uid),
     appId,
     authType,
     user