feat: 模型数据管理

feat: 模型数据导入

feat: redis 向量入库

feat: 向量索引

feat: 文件导入模型

perf: 交互

perf: prompt

src/pages/api/chat/chatGpt.ts

@@ -46,7 +46,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const model: ModelSchema = chat.modelId;
     const modelConstantsData = modelList.find((item) => item.model === model.service.modelName);
     if (!modelConstantsData) {
-      throw new Error('模型异常,请用 chatgpt 模型');
+      throw new Error('模型加载异常');
     }
 
     // 读取对话内容

src/pages/api/chat/vectorGpt.ts

@@ -0,0 +1,241 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { createParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser';
+import { connectToDatabase, ModelData } from '@/service/mongo';
+import { getOpenAIApi, authChat } from '@/service/utils/chat';
+import { httpsAgent, openaiChatFilter, systemPromptFilter } from '@/service/utils/tools';
+import { ChatCompletionRequestMessage, ChatCompletionRequestMessageRoleEnum } from 'openai';
+import { ChatItemType } from '@/types/chat';
+import { jsonRes } from '@/service/response';
+import type { ModelSchema } from '@/types/mongoSchema';
+import { PassThrough } from 'stream';
+import { modelList } from '@/constants/model';
+import { pushChatBill } from '@/service/events/pushBill';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataIndex } from '@/constants/redis';
+import { vectorToBuffer } from '@/utils/tools';
+
+let vectorData = [
+  -0.025028639, -0.010407282, 0.026523087, -0.0107438695, -0.006967359, 0.010043768, -0.012043097,
+  0.008724345, -0.028919589, -0.0117738275, 0.0050690062, 0.02961969
+].concat(new Array(1524).fill(0));
+
+/* 发送提示词 */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  let step = 0; // step=1时，表示开始了流响应
+  const stream = new PassThrough();
+  stream.on('error', () => {
+    console.log('error: ', 'stream error');
+    stream.destroy();
+  });
+  res.on('close', () => {
+    stream.destroy();
+  });
+  res.on('error', () => {
+    console.log('error: ', 'request error');
+    stream.destroy();
+  });
+
+  try {
+    const { chatId, prompt } = req.body as {
+      prompt: ChatItemType;
+      chatId: string;
+    };
+
+    const { authorization } = req.headers;
+    if (!chatId || !prompt) {
+      throw new Error('缺少参数');
+    }
+
+    await connectToDatabase();
+    const redis = await connectRedis();
+
+    const { chat, userApiKey, systemKey, userId } = await authChat(chatId, authorization);
+
+    const model: ModelSchema = chat.modelId;
+    const modelConstantsData = modelList.find((item) => item.model === model.service.modelName);
+    if (!modelConstantsData) {
+      throw new Error('模型加载异常');
+    }
+
+    // 读取对话内容
+    const prompts = [...chat.content, prompt];
+
+    // 获取 chatAPI
+    const chatAPI = getOpenAIApi(userApiKey || systemKey);
+
+    // 把输入的内容转成向量
+    const promptVector = await chatAPI
+      .createEmbedding(
+        {
+          model: 'text-embedding-ada-002',
+          input: prompt.value
+        },
+        {
+          timeout: 120000,
+          httpsAgent
+        }
+      )
+      .then((res) => res?.data?.data?.[0]?.embedding || []);
+
+    const binary = vectorToBuffer(promptVector);
+
+    // 搜索系统提示词, 按相似度从 redis 中搜出前3条不同 dataId 的数据
+    const redisData: any[] = await redis.sendCommand([
+      'FT.SEARCH',
+      `idx:${VecModelDataIndex}`,
+      `@modelId:{${String(chat.modelId._id)}} @vector:[VECTOR_RANGE 0.2 $blob]`,
+      // `@modelId:{${String(chat.modelId._id)}}=>[KNN 10 @vector $blob AS score]`,
+      'RETURN',
+      '1',
+      'dataId',
+      // 'SORTBY',
+      // 'score',
+      'PARAMS',
+      '2',
+      'blob',
+      binary,
+      'DIALECT',
+      '2'
+    ]);
+
+    // 格式化响应值，获取去重后的id
+    let formatIds = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
+      .map((i) => {
+        if (!redisData[i] || !redisData[i][1]) return '';
+        return redisData[i][1];
+      })
+      .filter((item) => item);
+    formatIds = Array.from(new Set(formatIds));
+
+    if (formatIds.length === 0) {
+      throw new Error('对不起，我没有找到你的问题');
+    }
+
+    // 从 mongo 中取出原文作为提示词
+    const textArr = (
+      await Promise.all(
+        [2, 4, 6, 8, 10, 12, 14, 16, 18, 20].map((i) => {
+          if (!redisData[i] || !redisData[i][1]) return '';
+          return ModelData.findById(redisData[i][1])
+            .select('text')
+            .then((res) => res?.text || '');
+        })
+      )
+    ).filter((item) => item);
+
+    // textArr 筛选，最多 3000 tokens
+    const systemPrompt = systemPromptFilter(textArr, 2800);
+
+    prompts.unshift({
+      obj: 'SYSTEM',
+      value: `请根据下面的知识回答问题： ${systemPrompt}`
+    });
+
+    // 控制在 tokens 数量，防止超出
+    const filterPrompts = openaiChatFilter(prompts, modelConstantsData.contextMaxToken);
+
+    // 格式化文本内容成 chatgpt 格式
+    const map = {
+      Human: ChatCompletionRequestMessageRoleEnum.User,
+      AI: ChatCompletionRequestMessageRoleEnum.Assistant,
+      SYSTEM: ChatCompletionRequestMessageRoleEnum.System
+    };
+    const formatPrompts: ChatCompletionRequestMessage[] = filterPrompts.map(
+      (item: ChatItemType) => ({
+        role: map[item.obj],
+        content: item.value
+      })
+    );
+    // console.log(formatPrompts);
+    // 计算温度
+    const temperature = modelConstantsData.maxTemperature * (model.temperature / 10);
+
+    let startTime = Date.now();
+    // 发出请求
+    const chatResponse = await chatAPI.createChatCompletion(
+      {
+        model: model.service.chatModel,
+        temperature: temperature,
+        // max_tokens: modelConstantsData.maxToken,
+        messages: formatPrompts,
+        frequency_penalty: 0.5, // 越大，重复内容越少
+        presence_penalty: -0.5, // 越大，越容易出现新内容
+        stream: true
+      },
+      {
+        timeout: 40000,
+        responseType: 'stream',
+        httpsAgent
+      }
+    );
+
+    console.log('api response time:', `${(Date.now() - startTime) / 1000}s`);
+
+    // 创建响应流
+    res.setHeader('Content-Type', 'text/event-stream;charset-utf-8');
+    res.setHeader('Access-Control-Allow-Origin', '*');
+    res.setHeader('X-Accel-Buffering', 'no');
+    res.setHeader('Cache-Control', 'no-cache, no-transform');
+    step = 1;
+
+    let responseContent = '';
+    stream.pipe(res);
+
+    const onParse = async (event: ParsedEvent | ReconnectInterval) => {
+      if (event.type !== 'event') return;
+      const data = event.data;
+      if (data === '[DONE]') return;
+      try {
+        const json = JSON.parse(data);
+        const content: string = json?.choices?.[0].delta.content || '';
+        if (!content || (responseContent === '' && content === '\n')) return;
+
+        responseContent += content;
+        // console.log('content:', content)
+        !stream.destroyed && stream.push(content.replace(/\n/g, '<br/>'));
+      } catch (error) {
+        error;
+      }
+    };
+
+    const decoder = new TextDecoder();
+    try {
+      for await (const chunk of chatResponse.data as any) {
+        if (stream.destroyed) {
+          // 流被中断了，直接忽略后面的内容
+          break;
+        }
+        const parser = createParser(onParse);
+        parser.feed(decoder.decode(chunk));
+      }
+    } catch (error) {
+      console.log('pipe error', error);
+    }
+    // close stream
+    !stream.destroyed && stream.push(null);
+    stream.destroy();
+
+    const promptsContent = formatPrompts.map((item) => item.content).join('');
+    // 只有使用平台的 key 才计费
+    pushChatBill({
+      isPay: !userApiKey,
+      modelName: model.service.modelName,
+      userId,
+      chatId,
+      text: promptsContent + responseContent
+    });
+    // jsonRes(res);
+  } catch (err: any) {
+    if (step === 1) {
+      // 直接结束流
+      console.log('error，结束');
+      stream.destroy();
+    } else {
+      res.status(500);
+      jsonRes(res, {
+        code: 500,
+        error: err
+      });
+    }
+  }
+}

src/pages/api/data/splitData.ts

@@ -24,7 +24,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     if (!DataRecord) {
       throw new Error('找不到数据集');
     }
-    const replaceText = text.replace(/[\r\n\\n]+/g, ' ');
+    const replaceText = text.replace(/[\\n]+/g, ' ');
 
     // 文本拆分成 chunk
     let chunks = replaceText.match(/[^!?.。]+[!?.。]/g) || [];
@@ -35,7 +35,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     chunks.forEach((chunk) => {
       splitText += chunk;
       const tokens = encode(splitText).length;
-      if (tokens >= 980) {
+      if (tokens >= 780) {
         dataItems.push({
           userId,
           dataId,

src/pages/api/model/create.ts

@@ -3,7 +3,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
-import { ModelStatusEnum, modelList, ChatModelNameEnum } from '@/constants/model';
+import { ModelStatusEnum, modelList, ChatModelNameEnum, ChatModelNameMap } from '@/constants/model';
 import { Model } from '@/service/models/model';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -33,15 +33,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     await connectToDatabase();
 
-    // 重名校验
-    const authRepeatName = await Model.findOne({
-      name,
-      userId
-    });
-    if (authRepeatName) {
-      throw new Error('模型名重复');
-    }
-
     // 上限校验
     const authCount = await Model.countDocuments({
       userId
@@ -57,9 +48,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       status: ModelStatusEnum.running,
       service: {
         company: modelItem.serviceCompany,
-        trainId: modelItem.trainName,
-        chatModel: modelItem.model,
-        modelName: modelItem.model
+        trainId: '',
+        chatModel: ChatModelNameMap[modelItem.model], // 聊天时用的模型
+        modelName: modelItem.model // 最底层的模型，不会变，用于计费等核心操作
       }
     });
 

src/pages/api/model/data/delModelDataById.ts

@@ -5,8 +5,8 @@ import { authToken } from '@/service/utils/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    let { modelId } = req.query as {
-      modelId: string;
+    let { dataId } = req.query as {
+      dataId: string;
     };
     const { authorization } = req.headers;
 
@@ -14,7 +14,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       throw new Error('无权操作');
     }
 
-    if (!modelId) {
+    if (!dataId) {
       throw new Error('缺少参数');
     }
 
@@ -24,7 +24,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     await connectToDatabase();
 
     await ModelData.deleteOne({
-      modelId,
+      _id: dataId,
       userId
     });
 

src/pages/api/model/data/getModelData.ts

@@ -14,6 +14,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       pageNum: string;
       pageSize: string;
     };
+
     const { authorization } = req.headers;
 
     pageNum = +pageNum;
@@ -41,7 +42,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       .limit(pageSize);
 
     jsonRes(res, {
-      data
+      data: {
+        pageNum,
+        pageSize,
+        data,
+        total: await ModelData.countDocuments({
+          modelId,
+          userId
+        })
+      }
     });
   } catch (err) {
     jsonRes(res, {

src/pages/api/model/data/pushModelDataInput.ts

@@ -2,12 +2,14 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, ModelData, Model } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
+import { ModelDataSchema } from '@/types/mongoSchema';
+import { generateVector } from '@/service/events/generateVector';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     const { modelId, data } = req.body as {
       modelId: string;
-      data: { q: string; a: string }[];
+      data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[];
     };
     const { authorization } = req.headers;
 
@@ -43,6 +45,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       }))
     );
 
+    generateVector(true);
+
     jsonRes(res, {
       data: model
     });

src/pages/api/model/data/pushModelDataSelectData.ts

@@ -0,0 +1,57 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, DataItem, ModelData } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    let { dataIds, modelId } = req.body as { dataIds: string[]; modelId: string };
+
+    if (!dataIds) {
+      throw new Error('参数错误');
+    }
+    await connectToDatabase();
+
+    const { authorization } = req.headers;
+
+    const userId = await authToken(authorization);
+
+    const dataItems = (
+      await Promise.all(
+        dataIds.map((dataId) =>
+          DataItem.find<{ _id: string; result: { q: string }[]; text: string }>(
+            {
+              userId,
+              dataId
+            },
+            'result text'
+          )
+        )
+      )
+    ).flat();
+
+    // push data
+    await ModelData.insertMany(
+      dataItems.map((item) => ({
+        modelId: modelId,
+        userId,
+        text: item.text,
+        q: item.result.map((item) => ({
+          id: nanoid(),
+          text: item.q
+        }))
+      }))
+    );
+
+    jsonRes(res, {
+      data: dataItems
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/putModelData.ts

@@ -5,9 +5,9 @@ import { authToken } from '@/service/utils/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    let { modelId, answer } = req.body as {
-      modelId: string;
-      answer: string;
+    let { dataId, text } = req.body as {
+      dataId: string;
+      text: string;
     };
     const { authorization } = req.headers;
 
@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       throw new Error('无权操作');
     }
 
-    if (!modelId) {
+    if (!dataId) {
       throw new Error('缺少参数');
     }
 
@@ -26,11 +26,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     await ModelData.updateOne(
       {
-        modelId,
+        _id: dataId,
         userId
       },
       {
-        a: answer
+        text
       }
     );
 

src/pages/api/model/data/splitData.ts

@@ -0,0 +1,67 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, SplitData, Model } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { generateQA } from '@/service/events/generateQA';
+import { encode } from 'gpt-token-utils';
+
+/* 拆分数据成QA */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { text, modelId } = req.body as { text: string; modelId: string };
+    if (!text || !modelId) {
+      throw new Error('参数错误');
+    }
+    await connectToDatabase();
+
+    const { authorization } = req.headers;
+
+    const userId = await authToken(authorization);
+
+    // 验证是否是该用户的 model
+    const model = await Model.findOne({
+      _id: modelId,
+      userId
+    });
+
+    if (!model) {
+      throw new Error('无权操作该模型');
+    }
+
+    const replaceText = text.replace(/(\\n|\n)+/g, ' ');
+
+    // 文本拆分成 chunk
+    let chunks = replaceText.match(/[^!?.。]+[!?.。]/g) || [];
+
+    const textList: string[] = [];
+    let splitText = '';
+
+    chunks.forEach((chunk) => {
+      splitText += chunk;
+      const tokens = encode(splitText).length;
+      if (tokens >= 980) {
+        textList.push(splitText);
+        splitText = '';
+      }
+    });
+
+    // 批量插入数据
+    await SplitData.create({
+      userId,
+      modelId,
+      rawText: text,
+      textList
+    });
+
+    // generateQA();
+
+    jsonRes(res, {
+      data: { chunks, replaceText }
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/del.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { Chat, Model, Training, connectToDatabase } from '@/service/mongo';
+import { Chat, Model, Training, connectToDatabase, ModelData } from '@/service/mongo';
 import { authToken, getUserOpenaiKey } from '@/service/utils/tools';
 import { TrainingStatusEnum } from '@/constants/model';
 import { getOpenAIApi } from '@/service/utils/chat';
@@ -26,16 +26,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     await connectToDatabase();
 
-    // 删除模型
-    await Model.deleteOne({
-      _id: modelId,
-      userId
-    });
-
+    let requestQueue: any[] = [];
     // 删除对应的聊天
-    await Chat.deleteMany({
-      modelId
-    });
+    requestQueue.push(
+      Chat.deleteMany({
+        modelId
+      })
+    );
+
+    // 删除数据集
+    requestQueue.push(
+      ModelData.deleteMany({
+        modelId
+      })
+    );
 
     // 查看是否正在训练
     const training: TrainingItemType | null = await Training.findOne({
@@ -56,9 +60,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     }
 
     // 删除对应训练记录
-    await Training.deleteMany({
-      modelId
-    });
+    requestQueue.push(
+      Training.deleteMany({
+        modelId
+      })
+    );
+
+    // 删除模型
+    requestQueue.push(
+      Model.deleteOne({
+        _id: modelId,
+        userId
+      })
+    );
+    await requestQueue;
 
     jsonRes(res);
   } catch (err) {

src/pages/api/model/update.ts

@@ -37,7 +37,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         systemPrompt,
         intro,
         temperature,
-        service,
+        // service,
         security
       }
     );