perf: 知识库数据结构

2023-04-01 22:31:56 +08:00
parent 5759cbeae0
commit ae4243b522
26 changed files with 611 additions and 518 deletions
--- a/src/pages/api/chat/vectorGpt.ts
+++ b/src/pages/api/chat/vectorGpt.ts
@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { createParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser';
-import { connectToDatabase, ModelData } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
 import { getOpenAIApi, authChat } from '@/service/utils/chat';
 import { httpsAgent, openaiChatFilter, systemPromptFilter } from '@/service/utils/tools';
 import { ChatCompletionRequestMessage, ChatCompletionRequestMessageRoleEnum } from 'openai';
@@ -11,7 +11,7 @@ import { PassThrough } from 'stream';
 import { modelList } from '@/constants/model';
 import { pushChatBill } from '@/service/events/pushBill';
 import { connectRedis } from '@/service/redis';
-import { VecModelDataIndex } from '@/constants/redis';
+import { VecModelDataPrefix } from '@/constants/redis';
 import { vectorToBuffer } from '@/utils/tools';

 /* 发送提示词 */
@@ -73,17 +73,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
      )
      .then((res) => res?.data?.data?.[0]?.embedding || []);

-    // 搜索系统提示词, 按相似度从 redis 中搜出前3条不同 dataId 的数据
+    // 搜索系统提示词, 按相似度从 redis 中搜出相关的 q 和 text
    const redisData: any[] = await redis.sendCommand([
      'FT.SEARCH',
-      `idx:${VecModelDataIndex}:hash`,
+      `idx:${VecModelDataPrefix}:hash`,
      `@modelId:{${String(
        chat.modelId._id
      )}} @vector:[VECTOR_RANGE 0.15 $blob]=>{$YIELD_DISTANCE_AS: score}`,
      // `@modelId:{${String(chat.modelId._id)}}=>[KNN 10 @vector $blob AS score]`,
      'RETURN',
      '1',
-      'dataId',
+      'text',
      'SORTBY',
      'score',
      'PARAMS',
@@ -97,42 +97,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
      '2'
    ]);

-    // 格式化响应值，获取去重后的id
-    let formatIds = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
+    // 格式化响应值，获取 qa
+    const formatRedisPrompt = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
      .map((i) => {
-        if (!redisData[i] || !redisData[i][1]) return '';
-        return redisData[i][1];
+        if (!redisData[i]) return '';
+        const text = (redisData[i][1] as string) || '';
+
+        if (!text) return '';
+
+        return text;
      })
      .filter((item) => item);
-    formatIds = Array.from(new Set(formatIds));

-    if (formatIds.length === 0) {
+    if (formatRedisPrompt.length === 0) {
      throw new Error('对不起，我没有找到你的问题');
    }

-    // 从 mongo 中取出原文作为提示词
-    const textArr = (
-      await Promise.all(
-        [2, 4, 6, 8, 10, 12, 14, 16, 18, 20].map((i) => {
-          if (!redisData[i] || !redisData[i][1]) return '';
-          return ModelData.findById(redisData[i][1])
-            .select('text q')
-            .then((res) => {
-              if (!res) return '';
-              // const questions = res.q.map((item) => item.text).join(' ');
-              const answer = res.text;
-              return `${answer}`;
-            });
-        })
-      )
-    ).filter((item) => item);
-
    // textArr 筛选，最多 3000 tokens
-    const systemPrompt = systemPromptFilter(textArr, 3400);
+    const systemPrompt = systemPromptFilter(formatRedisPrompt, 3400);

    prompts.unshift({
      obj: 'SYSTEM',
-      value: `${model.systemPrompt}。 我的知识库: "${systemPrompt}"`
+      value: `${model.systemPrompt} 我的知识库: "${systemPrompt}"`
    });

    // 控制在 tokens 数量，防止超出
--- a/src/pages/api/model/data/delModelDataById.ts
+++ b/src/pages/api/model/data/delModelDataById.ts
@@ -1,9 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, ModelData } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
 import { connectRedis } from '@/service/redis';
-import { VecModelDataIndex } from '@/constants/redis';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -23,25 +21,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    // 凭证校验
    const userId = await authToken(authorization);

-    await connectToDatabase();
    const redis = await connectRedis();

-    const data = await ModelData.findById(dataId);
-
-    await ModelData.deleteOne({
-      _id: dataId,
-      userId
-    });
-
-    // 删除 redis 数据
-    data?.q.forEach(async (item) => {
-      try {
-        await redis.json.del(`${VecModelDataIndex}:${item.id}`);
-      } catch (error) {
-        console.log(error);
-      }
-    });
-
+    // 校验是否为该用户的数据
+    const dataItemUserId = await redis.hGet(dataId, 'userId');
+    if (dataItemUserId !== userId) {
+      throw new Error('无权操作');
+    }
+    // 删除
+    await redis.del(dataId);
    jsonRes(res);
  } catch (err) {
    console.log(err);
--- a/src/pages/api/model/data/getModelData.ts
+++ b/src/pages/api/model/data/getModelData.ts
@@ -1,7 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, ModelData } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataIdx } from '@/constants/redis';
+import { SearchOptions } from 'redis';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -32,24 +35,34 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    const userId = await authToken(authorization);

    await connectToDatabase();
+    const redis = await connectRedis();

-    const data = await ModelData.find({
-      modelId,
-      userId
-    })
-      .sort({ _id: -1 }) // 按照创建时间倒序排列
-      .skip((pageNum - 1) * pageSize)
-      .limit(pageSize);
+    // 从 redis 中获取数据
+    const searchRes = await redis.ft.search(
+      VecModelDataIdx,
+      `@modelId:{${modelId}} @userId:{${userId}}`,
+      {
+        RETURN: ['q', 'text', 'status'],
+        LIMIT: {
+          from: (pageNum - 1) * pageSize,
+          size: pageSize
+        },
+        SORTBY: {
+          BY: 'modelId',
+          DIRECTION: 'DESC'
+        }
+      }
+    );

    jsonRes(res, {
      data: {
        pageNum,
        pageSize,
-        data,
-        total: await ModelData.countDocuments({
-          modelId,
-          userId
-        })
+        data: searchRes.documents.map((item) => ({
+          id: item.id,
+          ...item.value
+        })),
+        total: searchRes.total
      }
    });
  } catch (err) {
--- a/src/pages/api/model/data/pushModelDataInput.ts
+++ b/src/pages/api/model/data/pushModelDataInput.ts
@@ -1,9 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, ModelData, Model } from '@/service/mongo';
+import { connectToDatabase, Model } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
 import { ModelDataSchema } from '@/types/mongoSchema';
 import { generateVector } from '@/service/events/generateVector';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataPrefix, ModelDataStatusEnum } from '@/constants/redis';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -25,6 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    const userId = await authToken(authorization);

    await connectToDatabase();
+    const redis = await connectRedis();

    // 验证是否是该用户的 model
    const model = await Model.findOne({
@@ -36,19 +39,29 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
      throw new Error('无权操作该模型');
    }

-    // push data
-    await ModelData.insertMany(
-      data.map((item) => ({
-        ...item,
-        modelId,
-        userId
-      }))
+    const insertRes = await Promise.allSettled(
+      data.map((item) => {
+        return redis.sendCommand([
+          'HMSET',
+          `${VecModelDataPrefix}:${item.q.id}`,
+          'userId',
+          userId,
+          'modelId',
+          modelId,
+          'q',
+          item.q.text,
+          'text',
+          item.text,
+          'status',
+          ModelDataStatusEnum.waiting
+        ]);
+      })
    );

    generateVector(true);

    jsonRes(res, {
-      data: model
+      data: insertRes.filter((item) => item.status === 'rejected').length
    });
  } catch (err) {
    jsonRes(res, {
--- a/src/pages/api/model/data/pushModelDataJson.ts
+++ b/src/pages/api/model/data/pushModelDataJson.ts
@@ -0,0 +1,78 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Model } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { generateVector } from '@/service/events/generateVector';
+import { vectorToBuffer, formatVector } from '@/utils/tools';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataPrefix, ModelDataStatusEnum } from '@/constants/redis';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    const { modelId, data } = req.body as {
+      modelId: string;
+      data: { prompt: string; completion: string; vector?: number[] }[];
+    };
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+
+    if (!modelId || !Array.isArray(data)) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const userId = await authToken(authorization);
+
+    await connectToDatabase();
+    const redis = await connectRedis();
+
+    // 验证是否是该用户的 model
+    const model = await Model.findOne({
+      _id: modelId,
+      userId
+    });
+
+    if (!model) {
+      throw new Error('无权操作该模型');
+    }
+
+    // 插入 redis
+    const insertRedisRes = await Promise.allSettled(
+      data.map((item) => {
+        const vector = item.vector;
+
+        return redis.sendCommand([
+          'HMSET',
+          `${VecModelDataPrefix}:${nanoid()}`,
+          'userId',
+          userId,
+          'modelId',
+          String(modelId),
+          ...(vector ? ['vector', vectorToBuffer(formatVector(vector))] : []),
+          'q',
+          item.prompt,
+          'text',
+          item.completion,
+          'status',
+          vector ? ModelDataStatusEnum.ready : ModelDataStatusEnum.waiting
+        ]);
+      })
+    );
+
+    generateVector(true);
+
+    jsonRes(res, {
+      data: insertRedisRes.filter((item) => item.status === 'rejected').length
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}
--- a/src/pages/api/model/data/pushModelDataSelectData.ts
+++ b/src/pages/api/model/data/pushModelDataSelectData.ts
@@ -1,57 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, DataItem, ModelData } from '@/service/mongo';
-import { authToken } from '@/service/utils/tools';
-import { customAlphabet } from 'nanoid';
-const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    let { dataIds, modelId } = req.body as { dataIds: string[]; modelId: string };
-
-    if (!dataIds) {
-      throw new Error('参数错误');
-    }
-    await connectToDatabase();
-
-    const { authorization } = req.headers;
-
-    const userId = await authToken(authorization);
-
-    const dataItems = (
-      await Promise.all(
-        dataIds.map((dataId) =>
-          DataItem.find<{ _id: string; result: { q: string }[]; text: string }>(
-            {
-              userId,
-              dataId
-            },
-            'result text'
-          )
-        )
-      )
-    ).flat();
-
-    // push data
-    await ModelData.insertMany(
-      dataItems.map((item) => ({
-        modelId: modelId,
-        userId,
-        text: item.text,
-        q: item.result.map((item) => ({
-          id: nanoid(),
-          text: item.q
-        }))
-      }))
-    );
-
-    jsonRes(res, {
-      data: dataItems
-    });
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}
--- a/src/pages/api/model/data/putModelData.ts
+++ b/src/pages/api/model/data/putModelData.ts
@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, ModelData } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
+import { connectRedis } from '@/service/redis';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -22,17 +22,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    // 凭证校验
    const userId = await authToken(authorization);

-    await connectToDatabase();
+    const redis = await connectRedis();

-    await ModelData.updateOne(
-      {
-        _id: dataId,
-        userId
-      },
-      {
-        text
-      }
-    );
+    // 校验是否为该用户的数据
+    const dataItemUserId = await redis.hGet(dataId, 'userId');
+    if (dataItemUserId !== userId) {
+      throw new Error('无权操作');
+    }
+
+    // 更新
+    await redis.hSet(dataId, 'text', text);

    jsonRes(res);
  } catch (err) {
--- a/src/pages/api/model/del.ts
+++ b/src/pages/api/model/del.ts
@@ -1,13 +1,12 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { Chat, Model, Training, connectToDatabase, ModelData } from '@/service/mongo';
+import { Chat, Model, Training, connectToDatabase } from '@/service/mongo';
 import { authToken, getUserApiOpenai } from '@/service/utils/tools';
 import { TrainingStatusEnum } from '@/constants/model';
-import { getOpenAIApi } from '@/service/utils/chat';
 import { TrainingItemType } from '@/types/training';
 import { httpsAgent } from '@/service/utils/tools';
 import { connectRedis } from '@/service/redis';
-import { VecModelDataIndex } from '@/constants/redis';
+import { VecModelDataIdx } from '@/constants/redis';

 /* 获取我的模型 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -26,39 +25,38 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    // 凭证校验
    const userId = await authToken(authorization);

+    // 验证是否是该用户的 model
+    const model = await Model.findOne({
+      _id: modelId,
+      userId
+    });
+
+    if (!model) {
+      throw new Error('无权操作该模型');
+    }
+
    await connectToDatabase();
    const redis = await connectRedis();

-    const modelDataList = await ModelData.find({
+    // 获取 redis 中模型关联的所有数据
+    const searchRes = await redis.ft.search(
+      VecModelDataIdx,
+      `@modelId:{${modelId}} @userId:{${userId}}`,
+      {
+        LIMIT: {
+          from: 0,
+          size: 10000
+        }
+      }
+    );
+    // 删除 redis 内容
+    await Promise.all(searchRes.documents.map((item) => redis.del(item.id)));
+
+    // 删除对应的聊天
+    await Chat.deleteMany({
      modelId
    });

-    // 删除 redis
-    modelDataList?.forEach((modelData) =>
-      modelData.q.forEach(async (item) => {
-        try {
-          await redis.json.del(`${VecModelDataIndex}:${item.id}`);
-        } catch (error) {
-          console.log(error);
-        }
-      })
-    );
-
-    let requestQueue: any[] = [];
-    // 删除对应的聊天
-    requestQueue.push(
-      Chat.deleteMany({
-        modelId
-      })
-    );
-
-    // 删除数据集
-    requestQueue.push(
-      ModelData.deleteMany({
-        modelId
-      })
-    );
-
    // 查看是否正在训练
    const training: TrainingItemType | null = await Training.findOne({
      modelId,
@@ -78,21 +76,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    }

    // 删除对应训练记录
-    requestQueue.push(
-      Training.deleteMany({
-        modelId
-      })
-    );
+    await Training.deleteMany({
+      modelId
+    });

    // 删除模型
-    requestQueue.push(
-      Model.deleteOne({
-        _id: modelId,
-        userId
-      })
-    );
-
-    await Promise.all(requestQueue);
+    await Model.deleteOne({
+      _id: modelId,
+      userId
+    });

    jsonRes(res);
  } catch (err) {
--- a/src/pages/api/timer/testVector.ts
+++ b/src/pages/api/timer/testVector.ts
@@ -1,68 +0,0 @@
-// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, Bill } from '@/service/mongo';
-import { authToken } from '@/service/utils/tools';
-import type { BillSchema } from '@/types/mongoSchema';
-import { VecModelDataIndex } from '@/constants/redis';
-import { connectRedis } from '@/service/redis';
-import { vectorToBuffer } from '@/utils/tools';
-
-let vectorData = [
-  -0.025028639, -0.010407282, 0.026523087, -0.0107438695, -0.006967359, 0.010043768, -0.012043097,
-  0.008724345, -0.028919589, -0.0117738275, 0.0050690062, 0.02961969
-].concat(new Array(1524).fill(0));
-let vectorData2 = [
-  0.025028639, 0.010407282, 0.026523087, 0.0107438695, -0.006967359, 0.010043768, -0.012043097,
-  0.008724345, 0.028919589, 0.0117738275, 0.0050690062, 0.02961969
-].concat(new Array(1524).fill(0));
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    if (process.env.NODE_ENV !== 'development') {
-      throw new Error('不是开发环境');
-    }
-    await connectToDatabase();
-
-    const redis = await connectRedis();
-
-    await redis.sendCommand([
-      'HMSET',
-      'model:data:333',
-      'vector',
-      vectorToBuffer(vectorData2),
-      'modelId',
-      '1133',
-      'dataId',
-      'safadfa'
-    ]);
-
-    // search
-    const response = await redis.sendCommand([
-      'FT.SEARCH',
-      'idx:model:data:hash',
-      '@modelId:{1133} @vector:[VECTOR_RANGE 0.15 $blob]=>{$YIELD_DISTANCE_AS: score}',
-      'RETURN',
-      '2',
-      'modelId',
-      'dataId',
-      'PARAMS',
-      '2',
-      'blob',
-      vectorToBuffer(vectorData2),
-      'SORTBY',
-      'score',
-      'DIALECT',
-      '2'
-    ]);
-
-    jsonRes(res, {
-      data: response
-    });
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}
--- a/src/pages/chat/index.tsx
+++ b/src/pages/chat/index.tsx
@@ -190,97 +190,91 @@ const Chat = ({ chatId }: { chatId: string }) => {
  /**
   * 发送一个内容
   */
-  const sendPrompt = useCallback(
-    async (e?: React.MouseEvent<HTMLDivElement>) => {
-      e?.stopPropagation();
-      e?.preventDefault();
+  const sendPrompt = useCallback(async () => {
+    const storeInput = inputVal;
+    // 去除空行
+    const val = inputVal
+      .trim()
+      .split('\n')
+      .filter((val) => val)
+      .join('\n');
+    if (!chatData?.modelId || !val || !ChatBox.current || isChatting) {
+      return;
+    }

-      const storeInput = inputVal;
-      // 去除空行
-      const val = inputVal
-        .trim()
-        .split('\n')
-        .filter((val) => val)
-        .join('\n');
-      if (!chatData?.modelId || !val || !ChatBox.current || isChatting) {
-        return;
+    // 长度校验
+    const tokens = encode(val).length;
+    const model = modelList.find((item) => item.model === chatData.modelName);
+
+    if (model && tokens >= model.maxToken) {
+      toast({
+        title: '单次输入超出 4000 tokens',
+        status: 'warning'
+      });
+      return;
+    }
+
+    const newChatList: ChatSiteItemType[] = [
+      ...chatData.history,
+      {
+        obj: 'Human',
+        value: val,
+        status: 'finish'
+      },
+      {
+        obj: 'AI',
+        value: '',
+        status: 'loading'
      }
+    ];

-      // 长度校验
-      const tokens = encode(val).length;
-      const model = modelList.find((item) => item.model === chatData.modelName);
+    // 插入内容
+    setChatData((state) => ({
+      ...state,
+      history: newChatList
+    }));

-      if (model && tokens >= model.maxToken) {
-        toast({
-          title: '单次输入超出 4000 tokens',
-          status: 'warning'
+    // 清空输入内容
+    resetInputVal('');
+    scrollToBottom();
+
+    try {
+      await gptChatPrompt(newChatList[newChatList.length - 2]);
+
+      // 如果是 Human 第一次发送，插入历史记录
+      const humanChat = newChatList.filter((item) => item.obj === 'Human');
+      if (humanChat.length === 1) {
+        pushChatHistory({
+          chatId,
+          title: humanChat[0].value
        });
-        return;
      }
+    } catch (err: any) {
+      toast({
+        title: typeof err === 'string' ? err : err?.message || '聊天出错了~',
+        status: 'warning',
+        duration: 5000,
+        isClosable: true
+      });

-      const newChatList: ChatSiteItemType[] = [
-        ...chatData.history,
-        {
-          obj: 'Human',
-          value: val,
-          status: 'finish'
-        },
-        {
-          obj: 'AI',
-          value: '',
-          status: 'loading'
-        }
-      ];
+      resetInputVal(storeInput);

-      // 插入内容
      setChatData((state) => ({
        ...state,
-        history: newChatList
+        history: newChatList.slice(0, newChatList.length - 2)
      }));
-
-      // 清空输入内容
-      resetInputVal('');
-      scrollToBottom();
-
-      try {
-        await gptChatPrompt(newChatList[newChatList.length - 2]);
-
-        // 如果是 Human 第一次发送，插入历史记录
-        const humanChat = newChatList.filter((item) => item.obj === 'Human');
-        if (humanChat.length === 1) {
-          pushChatHistory({
-            chatId,
-            title: humanChat[0].value
-          });
-        }
-      } catch (err: any) {
-        toast({
-          title: typeof err === 'string' ? err : err?.message || '聊天出错了~',
-          status: 'warning',
-          duration: 5000,
-          isClosable: true
-        });
-
-        resetInputVal(storeInput);
-
-        setChatData((state) => ({
-          ...state,
-          history: newChatList.slice(0, newChatList.length - 2)
-        }));
-      }
-    },
-    [
-      inputVal,
-      chatData,
-      isChatting,
-      resetInputVal,
-      scrollToBottom,
-      toast,
-      gptChatPrompt,
-      pushChatHistory,
-      chatId
-    ]
-  );
+    }
+  }, [
+    inputVal,
+    chatData,
+    isChatting,
+    resetInputVal,
+    scrollToBottom,
+    toast,
+    gptChatPrompt,
+    pushChatHistory,
+    chatId
+  ]);

  // 删除一句话
  const delChatRecord = useCallback(
@@ -474,6 +468,7 @@ const Chat = ({ chatId }: { chatId: string }) => {
              flex={1}
              w={0}
              py={0}
+              pr={0}
              border={'none'}
              _focusVisible={{
                border: 'none'
--- a/src/pages/model/detail/components/InputDataModal.tsx
+++ b/src/pages/model/detail/components/InputDataModal.tsx
@@ -45,24 +45,22 @@ const InputDataModal = ({
      setImporting(true);

      try {
-        await postModelDataInput({
+        const res = await postModelDataInput({
          modelId: modelId,
          data: [
            {
              text: e.text,
-              q: [
-                {
-                  id: nanoid(),
-                  text: e.q
-                }
-              ]
+              q: {
+                id: nanoid(),
+                text: e.q
+              }
            }
          ]
        });

        toast({
-          title: '导入数据成功,需要一段时间训练',
-          status: 'success'
+          title: res === 0 ? '导入数据成功,需要一段时间训练' : '数据导入异常',
+          status: res === 0 ? 'success' : 'warning'
        });
        onClose();
        onSuccess();
@@ -88,8 +86,15 @@ const InputDataModal = ({
        <ModalHeader>手动导入</ModalHeader>
        <ModalCloseButton />

-        <Flex flex={'1 0 0'} h={0} px={6} pb={2}>
-          <Box flex={2} mr={4} h={'100%'}>
+        <Box
+          display={['block', 'flex']}
+          flex={'1 0 0'}
+          h={['100%', 0]}
+          overflowY={'auto'}
+          px={6}
+          pb={2}
+        >
+          <Box flex={2} mr={[0, 4]} mb={[4, 0]} h={['230px', '100%']}>
            <Box h={'30px'}>问题</Box>
            <Textarea
              placeholder="相关问题，可以回车输入多个问法, 最多500字"
@@ -101,10 +106,11 @@ const InputDataModal = ({
              })}
            />
          </Box>
-          <Box flex={3} h={'100%'}>
+          <Box flex={3} h={['330px', '100%']}>
            <Box h={'30px'}>知识点</Box>
            <Textarea
-              placeholder="知识点"
+              placeholder="知识点,最多1000字"
+              maxLength={1000}
              resize={'none'}
              h={'calc(100% - 30px)'}
              {...register(`text`, {
@@ -112,7 +118,7 @@ const InputDataModal = ({
              })}
            />
          </Box>
-        </Flex>
+        </Box>

        <Flex px={6} pt={2} pb={4}>
          <Box flex={1}></Box>
--- a/src/pages/model/detail/components/ModelDataCard.tsx
+++ b/src/pages/model/detail/components/ModelDataCard.tsx
@@ -19,7 +19,7 @@ import {
  MenuItem
 } from '@chakra-ui/react';
 import type { ModelSchema } from '@/types/mongoSchema';
-import { ModelDataSchema } from '@/types/mongoSchema';
+import type { RedisModelDataItemType } from '@/types/redis';
 import { ModelDataStatusMap } from '@/constants/model';
 import { usePagination } from '@/hooks/usePagination';
 import {
@@ -35,7 +35,8 @@ import dynamic from 'next/dynamic';
 import { useQuery } from '@tanstack/react-query';

 const InputModel = dynamic(() => import('./InputDataModal'));
-const SelectModel = dynamic(() => import('./SelectFileModal'));
+const SelectFileModel = dynamic(() => import('./SelectFileModal'));
+const SelectJsonModel = dynamic(() => import('./SelectJsonModal'));

 const ModelDataCard = ({ model }: { model: ModelSchema }) => {
  const { toast } = useToast();
@@ -48,7 +49,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
    total,
    getData,
    pageNum
-  } = usePagination<ModelDataSchema>({
+  } = usePagination<RedisModelDataItemType>({
    api: getModelDataList,
    pageSize: 8,
    params: {
@@ -76,12 +77,17 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
    onClose: onCloseInputModal
  } = useDisclosure();
  const {
-    isOpen: isOpenSelectModal,
-    onOpen: onOpenSelectModal,
-    onClose: onCloseSelectModal
+    isOpen: isOpenSelectFileModal,
+    onOpen: onOpenSelectFileModal,
+    onClose: onCloseSelectFileModal
+  } = useDisclosure();
+  const {
+    isOpen: isOpenSelectJsonModal,
+    onOpen: onOpenSelectJsonModal,
+    onClose: onCloseSelectJsonModal
  } = useDisclosure();

-  const { data, refetch } = useQuery(['getModelSplitDataList'], () =>
+  const { data: splitDataList, refetch } = useQuery(['getModelSplitDataList'], () =>
    getModelSplitDataList(model._id)
  );

@@ -113,13 +119,18 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
          <MenuButton as={Button}>导入</MenuButton>
          <MenuList>
            <MenuItem onClick={onOpenInputModal}>手动输入</MenuItem>
-            <MenuItem onClick={onOpenSelectModal}>文件导入</MenuItem>
+            <MenuItem onClick={onOpenSelectFileModal}>文件导入</MenuItem>
+            <MenuItem onClick={onOpenSelectJsonModal}>JSON导入</MenuItem>
          </MenuList>
        </Menu>
      </Flex>
-      {data && data.length > 0 && <Box fontSize={'xs'}>{data.length}条数据正在拆分中...</Box>}
+      {splitDataList && splitDataList.length > 0 && (
+        <Box fontSize={'xs'}>
+          {splitDataList.map((item) => item.textList).flat().length}条数据正在拆分...
+        </Box>
+      )}
      <Box mt={4}>
-        <TableContainer>
+        <TableContainer minH={'500px'}>
          <Table variant={'simple'}>
            <Thead>
              <Tr>
@@ -131,19 +142,11 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
            </Thead>
            <Tbody>
              {modelDataList.map((item) => (
-                <Tr key={item._id}>
+                <Tr key={item.id}>
                  <Td w={'350px'}>
-                    {item.q.map((item, i) => (
-                      <Box
-                        key={item.id}
-                        fontSize={'xs'}
-                        w={'100%'}
-                        whiteSpace={'pre-wrap'}
-                        _notLast={{ mb: 1 }}
-                      >
-                        {item.text}
-                      </Box>
-                    ))}
+                    <Box fontSize={'xs'} w={'100%'} whiteSpace={'pre-wrap'} _notLast={{ mb: 1 }}>
+                      {item.q}
+                    </Box>
                  </Td>
                  <Td minW={'200px'}>
                    <Textarea
@@ -153,9 +156,9 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
                      fontSize={'xs'}
                      resize={'both'}
                      onBlur={(e) => {
-                        const oldVal = modelDataList.find((data) => item._id === data._id)?.text;
+                        const oldVal = modelDataList.find((data) => item.id === data.id)?.text;
                        if (oldVal !== e.target.value) {
-                          updateAnswer(item._id, e.target.value);
+                          updateAnswer(item.id, e.target.value);
                        }
                      }}
                    ></Textarea>
@@ -169,7 +172,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
                      aria-label={'delete'}
                      size={'sm'}
                      onClick={async () => {
-                        await delOneModelData(item._id);
+                        await delOneModelData(item.id);
                        refetchData(pageNum);
                      }}
                    />
@@ -188,8 +191,19 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
      {isOpenInputModal && (
        <InputModel modelId={model._id} onClose={onCloseInputModal} onSuccess={refetchData} />
      )}
-      {isOpenSelectModal && (
-        <SelectModel modelId={model._id} onClose={onCloseSelectModal} onSuccess={refetchData} />
+      {isOpenSelectFileModal && (
+        <SelectFileModel
+          modelId={model._id}
+          onClose={onCloseSelectFileModal}
+          onSuccess={refetchData}
+        />
+      )}
+      {isOpenSelectJsonModal && (
+        <SelectJsonModel
+          modelId={model._id}
+          onClose={onCloseSelectJsonModal}
+          onSuccess={refetchData}
+        />
      )}
    </>
  );
--- a/src/pages/model/detail/components/SelectFileModal.tsx
+++ b/src/pages/model/detail/components/SelectFileModal.tsx
@@ -100,40 +100,43 @@ const SelectFileModal = ({
  });

  return (
-    <Modal isOpen={true} onClose={onClose}>
+    <Modal isOpen={true} onClose={onClose} isCentered>
      <ModalOverlay />
-      <ModalContent maxW={'min(900px, 90vw)'} position={'relative'}>
+      <ModalContent maxW={'min(900px, 90vw)'} m={0} position={'relative'} h={['90vh', '70vh']}>
        <ModalHeader>文件导入</ModalHeader>
        <ModalCloseButton />

-        <ModalBody>
-          <Flex
-            flexDirection={'column'}
+        <ModalBody
+          display={'flex'}
+          flexDirection={'column'}
+          p={4}
+          h={'100%'}
+          alignItems={'center'}
+          justifyContent={'center'}
+          fontSize={'sm'}
+        >
+          <Button isLoading={selecting} onClick={onOpen}>
+            选择文件
+          </Button>
+          <Box mt={2} maxW={['100%', '70%']}>
+            支持 {fileExtension} 文件。模型会自动对文本进行 QA 拆分，需要较长训练时间，拆分需要消耗
+            tokens，大约0.04元/1k tokens，请确保账号余额充足。
+          </Box>
+          <Box mt={2}>
+            一共 {fileText.length} 个字，{encode(fileText).length} 个tokens
+          </Box>
+          <Box
+            flex={'1 0 0'}
+            h={0}
+            w={'100%'}
+            overflowY={'auto'}
            p={2}
-            h={'100%'}
-            alignItems={'center'}
-            justifyContent={'center'}
-            fontSize={'sm'}
+            backgroundColor={'blackAlpha.50'}
+            whiteSpace={'pre-wrap'}
+            fontSize={'xs'}
          >
-            <Button isLoading={selecting} onClick={onOpen}>
-              选择文件
-            </Button>
-            <Box mt={2}>支持 {fileExtension} 文件. 会先对文本进行拆分，需要时间较长。</Box>
-            <Box mt={2}>
-              一共 {fileText.length} 个字，{encode(fileText).length} 个tokens
-            </Box>
-            <Box
-              h={'300px'}
-              w={'100%'}
-              overflow={'auto'}
-              p={2}
-              backgroundColor={'blackAlpha.50'}
-              whiteSpace={'pre'}
-              fontSize={'xs'}
-            >
-              {fileText}
-            </Box>
-          </Flex>
+            {fileText}
+          </Box>
        </ModalBody>

        <Flex px={6} pt={2} pb={4}>
--- a/src/pages/model/detail/components/SelectJsonModal.tsx
+++ b/src/pages/model/detail/components/SelectJsonModal.tsx
@@ -0,0 +1,145 @@
+import React, { useState, useCallback } from 'react';
+import {
+  Box,
+  Flex,
+  Button,
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody
+} from '@chakra-ui/react';
+import { useToast } from '@/hooks/useToast';
+import { useSelectFile } from '@/hooks/useSelectFile';
+import { useConfirm } from '@/hooks/useConfirm';
+import { readTxtContent } from '@/utils/tools';
+import { useMutation } from '@tanstack/react-query';
+import { postModelDataJsonData } from '@/api/model';
+import Markdown from '@/components/Markdown';
+
+const SelectJsonModal = ({
+  onClose,
+  onSuccess,
+  modelId
+}: {
+  onClose: () => void;
+  onSuccess: () => void;
+  modelId: string;
+}) => {
+  const [selecting, setSelecting] = useState(false);
+  const { toast } = useToast();
+  const { File, onOpen } = useSelectFile({ fileType: '.json', multiple: true });
+  const [fileData, setFileData] = useState<
+    { prompt: string; completion: string; vector?: number[] }[]
+  >([]);
+  const { openConfirm, ConfirmChild } = useConfirm({
+    content: '确认导入该数据集?'
+  });
+
+  const onSelectFile = useCallback(
+    async (e: File[]) => {
+      setSelecting(true);
+      try {
+        const jsonData = (
+          await Promise.all(e.map((item) => readTxtContent(item).then((text) => JSON.parse(text))))
+        ).flat();
+        // check 文件类型
+        for (let i = 0; i < jsonData.length; i++) {
+          if (!jsonData[i]?.prompt || !jsonData[i]?.completion) {
+            throw new Error('缺少 prompt 或 completion');
+          }
+        }
+
+        setFileData(jsonData);
+      } catch (error: any) {
+        console.log(error);
+        toast({
+          title: error?.message || 'JSON文件格式有误',
+          status: 'error'
+        });
+      }
+      setSelecting(false);
+    },
+    [setSelecting, toast]
+  );
+
+  const { mutate, isLoading } = useMutation({
+    mutationFn: async () => {
+      if (!fileData) return;
+      const res = await postModelDataJsonData(modelId, fileData);
+      console.log(res);
+      toast({
+        title: '导入数据成功,需要一段拆解和训练',
+        status: 'success'
+      });
+      onClose();
+      onSuccess();
+    },
+    onError() {
+      toast({
+        title: '导入文件失败',
+        status: 'error'
+      });
+    }
+  });
+
+  return (
+    <Modal isOpen={true} onClose={onClose} isCentered>
+      <ModalOverlay />
+      <ModalContent maxW={'90vw'} position={'relative'} m={0} h={'90vh'}>
+        <ModalHeader>JSON数据集</ModalHeader>
+        <ModalCloseButton />
+
+        <ModalBody h={'100%'} display={['block', 'flex']} fontSize={'sm'} overflowY={'auto'}>
+          <Box flex={'2 0 0'} w={['100%', 0]} mr={[0, 4]} mb={[4, 0]}>
+            <Markdown
+              source={`接受一个对象数组，每个对象必须包含 prompt 和 completion 格式，可以包含vector。prompt 代表问题，completion 代表回答的内容，可以多个问题对应一个回答，vector 为 prompt 的向量，如果没有讲有系统生成。例如：
+~~~json
+[
+  {
+    "prompt":"sealos是什么?\\n介绍下sealos\\nsealos有什么用",
+    "completion":"sealos是xxxxxx"
+  },
+  {
+    "prompt":"laf是什么?",
+    "completion":"laf是xxxxxx",
+    "vector":[-0.42,-0.4314314,0.43143]
+  }
+]
+~~~`}
+            />
+            <Flex alignItems={'center'}>
+              <Button isLoading={selecting} onClick={onOpen}>
+                选择 JSON 数据集
+              </Button>
+
+              <Box ml={4}>一共 {fileData.length} 组数据</Box>
+            </Flex>
+          </Box>
+          <Box flex={'2 0 0'} h={'100%'} overflow={'auto'} p={2} backgroundColor={'blackAlpha.50'}>
+            {JSON.stringify(fileData)}
+          </Box>
+        </ModalBody>
+
+        <Flex px={6} pt={2} pb={4}>
+          <Box flex={1}></Box>
+          <Button variant={'outline'} mr={3} onClick={onClose}>
+            取消
+          </Button>
+          <Button
+            isLoading={isLoading}
+            isDisabled={fileData.length === 0}
+            onClick={openConfirm(mutate)}
+          >
+            确认导入
+          </Button>
+        </Flex>
+      </ModalContent>
+      <ConfirmChild />
+      <File onSelect={onSelectFile} />
+    </Modal>
+  );
+};
+
+export default SelectJsonModal;