From 971c9cb291136ded7bec2ff81c85a154e6c9eabb Mon Sep 17 00:00:00 2001
From: archer <545436317@qq.com>
Date: Thu, 7 Sep 2023 17:30:04 +0800
Subject: [PATCH] perf: qa default value

---
 README.md                                     |  2 +-
 client/src/pages/api/openapi/plugin/vector.ts |  1 +
 .../components/Import/CreateFileModal.tsx     |  2 +-
 client/src/service/events/generateQA.ts       | 12 +++++++++
 client/src/service/response.ts                |  4 +--
 client/src/utils/plugin/openai.ts             | 26 ++++++++++++++-----
 6 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 15e7dd313..a5c18b8c1 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,7 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
    - [x] 支持 url 读取、 CSV 批量导入
    - [x] 支持知识库单独设置向量模型
    - [x] 源文件存储
-   - [x] 文件学习 Agent
+   - [ ] 文件学习 Agent
 3. 多种效果测试渠道
    - [x] 知识库单点搜索测试
    - [x] 对话时反馈引用并可修改与删除
diff --git a/client/src/pages/api/openapi/plugin/vector.ts b/client/src/pages/api/openapi/plugin/vector.ts
index 5ea7f324e..39f2f0dd0 100644
--- a/client/src/pages/api/openapi/plugin/vector.ts
+++ b/client/src/pages/api/openapi/plugin/vector.ts
@@ -68,6 +68,7 @@ export async function getVector({
     )
     .then(async (res) => {
       if (!res.data?.data?.[0]?.embedding) {
+        console.log(res.data);
         // @ts-ignore
         return Promise.reject(res.data?.error?.message || 'Embedding API Error');
       }
diff --git a/client/src/pages/kb/detail/components/Import/CreateFileModal.tsx b/client/src/pages/kb/detail/components/Import/CreateFileModal.tsx
index 213d18cbb..8e7d9ba4e 100644
--- a/client/src/pages/kb/detail/components/Import/CreateFileModal.tsx
+++ b/client/src/pages/kb/detail/components/Import/CreateFileModal.tsx
@@ -20,7 +20,7 @@ const CreateFileModal = ({
   });
 
   return (
-    <MyModal title={t('file.Create File')} isOpen onClose={onClose} w={'600px'} top={'15vh'}>
+    <MyModal title={t('file.Create File')} isOpen onClose={() => {}} w={'600px'} top={'15vh'}>
       <ModalBody>
         <Box mb={1} fontSize={'sm'}>
           文件名
diff --git a/client/src/service/events/generateQA.ts b/client/src/service/events/generateQA.ts
index 58dbde75d..efdb43e1e 100644
--- a/client/src/service/events/generateQA.ts
+++ b/client/src/service/events/generateQA.ts
@@ -10,6 +10,7 @@ import { ChatCompletionRequestMessage } from 'openai';
 import { modelToolMap } from '@/utils/plugin';
 import { gptMessage2ChatType } from '@/utils/adapt';
 import { addLog } from '../utils/tools';
+import { splitText2Chunks } from '@/utils/file';
 
 const reduceQueue = () => {
   global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -212,5 +213,16 @@ function formatSplitText(text: string) {
     }
   }
 
+  // empty result. direct split chunk
+  if (result.length === 0) {
+    const splitRes = splitText2Chunks({ text: text, maxLen: 500 });
+    splitRes.chunks.forEach((item) => {
+      result.push({
+        q: item,
+        a: ''
+      });
+    });
+  }
+
   return result;
 }
diff --git a/client/src/service/response.ts b/client/src/service/response.ts
index 830a4bb6a..c6ef051a2 100644
--- a/client/src/service/response.ts
+++ b/client/src/service/response.ts
@@ -44,7 +44,7 @@ export const jsonRes = <T = any>(
     if (typeof error === 'string') {
       msg = error;
     } else if (proxyError[error?.code]) {
-      msg = '接口连接异常';
+      msg = '网络连接异常';
     } else if (error?.response?.data?.error?.message) {
       msg = error?.response?.data?.error?.message;
     } else if (openaiAccountError[error?.response?.data?.error?.code]) {
@@ -85,7 +85,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => {
   if (typeof error === 'string') {
     msg = error;
   } else if (proxyError[error?.code]) {
-    msg = '接口连接异常';
+    msg = '网络连接异常';
   } else if (error?.response?.data?.error?.message) {
     msg = error?.response?.data?.error?.message;
   } else if (openaiAccountError[error?.response?.data?.error?.code]) {
diff --git a/client/src/utils/plugin/openai.ts b/client/src/utils/plugin/openai.ts
index 2558ea26c..0e8038233 100644
--- a/client/src/utils/plugin/openai.ts
+++ b/client/src/utils/plugin/openai.ts
@@ -51,9 +51,18 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
   const adaptMessages = adaptChatItem_openAI({ messages, reserveId: true });
   const token = adaptMessages.reduce((sum, item) => {
     const text = `${item.role}\n${item.content}`;
-    const enc = getOpenAiEncMap();
-    const encodeText = enc.encode(text);
-    const tokens = encodeText.length + 3; // 补充估算值
+
+    /* use textLen as tokens if encode error */
+    const tokens = (() => {
+      try {
+        const enc = getOpenAiEncMap();
+        const encodeText = enc.encode(text);
+        return encodeText.length + 3; // 补充估算值
+      } catch (error) {
+        return text.length;
+      }
+    })();
+
     return sum + tokens;
   }, 0);
 
@@ -62,9 +71,14 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
 
 export const openAiSliceTextByToken = ({ text, length }: { text: string; length: number }) => {
   const enc = getOpenAiEncMap();
-  const encodeText = enc.encode(text);
-  const decoder = new TextDecoder();
-  return decoder.decode(enc.decode(encodeText.slice(0, length)));
+
+  try {
+    const encodeText = enc.encode(text);
+    const decoder = new TextDecoder();
+    return decoder.decode(enc.decode(encodeText.slice(0, length)));
+  } catch (error) {
+    return text.slice(0, length);
+  }
 };
 
 export const authOpenAiKey = async (key: string) => {