perf: code

src/pages/api/openapi/kb/appKbSearch.ts

@@ -10,7 +10,6 @@ import { authModel } from '@/service/utils/auth';
 import { ChatModelMap } from '@/constants/model';
 import { ChatRoleEnum } from '@/constants/chat';
 import { openaiEmbedding } from '../plugin/openaiEmbedding';
-import { ModelDataStatusEnum } from '@/constants/model';
 import { modelToolMap } from '@/utils/plugin';
 
 export type QuoteItemType = { id: string; q: string; a: string; isEdit: boolean };
@@ -102,8 +101,6 @@ export async function appKbSearch({
       PgClient.select<QuoteItemType>('modelData', {
         fields: ['id', 'q', 'a'],
         where: [
-          ['status', ModelDataStatusEnum.ready],
-          'AND',
           `kb_id IN (${model.chat.relatedKbs.map((item) => `'${item}'`).join(',')})`,
           'AND',
           `vector <=> '[${promptVector}]' < ${similarity}`

src/pages/api/openapi/kb/pushData.ts

@@ -5,13 +5,13 @@ import { connectToDatabase, TrainingData } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { authKb } from '@/service/utils/auth';
 import { withNextCors } from '@/service/utils/tools';
-import { TrainingTypeEnum } from '@/constants/plugin';
+import { TrainingModeEnum } from '@/constants/plugin';
 import { startQueue } from '@/service/utils/tools';
 
 export type Props = {
   kbId: string;
   data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
-  mode: `${TrainingTypeEnum}`;
+  mode: `${TrainingModeEnum}`;
   prompt?: string;
 };
 
@@ -60,6 +60,39 @@ export async function pushDataToKb({
     return {};
   }
 
+  // 去重
+  // 过滤重复的 qa 内容
+  // const searchRes = await Promise.allSettled(
+  //   dataItems.map(async ({ q, a = '' }) => {
+  //     if (!q) {
+  //       return Promise.reject('q为空');
+  //     }
+
+  //     q = q.replace(/\\n/g, '\n');
+  //     a = a.replace(/\\n/g, '\n');
+
+  //     // Exactly the same data, not push
+  //     try {
+  //       const count = await PgClient.count('modelData', {
+  //         where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['q', q], 'AND', ['a', a]]
+  //       });
+
+  //       if (count > 0) {
+  //         return Promise.reject('已经存在');
+  //       }
+  //     } catch (error) {
+  //       error;
+  //     }
+  //     return Promise.resolve({
+  //       q,
+  //       a
+  //     });
+  //   })
+  // );
+  // const filterData = searchRes
+  //   .filter((item) => item.status === 'fulfilled')
+  //   .map<{ q: string; a: string }>((item: any) => item.value);
+
   // 插入记录
   await TrainingData.insertMany(
     data.map((item) => ({

src/pages/api/openapi/kb/updateData.ts

@@ -1,8 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
-import { ModelDataStatusEnum } from '@/constants/model';
-import { generateVector } from '@/service/events/generateVector';
 import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
 import { openaiEmbedding } from '../plugin/openaiEmbedding';