perf: qa default value

README.md

@@ -48,7 +48,7 @@ FastGPT 是一个基于 LLM 大语言模型的知识库问答系统，提供开
    - [x] 支持 url 读取、 CSV 批量导入
    - [x] 支持知识库单独设置向量模型
    - [x] 源文件存储
-   - [x] 文件学习 Agent
+   - [ ] 文件学习 Agent
 3. 多种效果测试渠道
    - [x] 知识库单点搜索测试
    - [x] 对话时反馈引用并可修改与删除

client/src/pages/api/openapi/plugin/vector.ts

@@ -68,6 +68,7 @@ export async function getVector({
     )
     .then(async (res) => {
       if (!res.data?.data?.[0]?.embedding) {
+        console.log(res.data);
         // @ts-ignore
         return Promise.reject(res.data?.error?.message || 'Embedding API Error');
       }

client/src/pages/kb/detail/components/Import/CreateFileModal.tsx

@@ -20,7 +20,7 @@ const CreateFileModal = ({
   });
 
   return (
-    <MyModal title={t('file.Create File')} isOpen onClose={onClose} w={'600px'} top={'15vh'}>
+    <MyModal title={t('file.Create File')} isOpen onClose={() => {}} w={'600px'} top={'15vh'}>
       <ModalBody>
         <Box mb={1} fontSize={'sm'}>
           文件名

client/src/service/events/generateQA.ts

@@ -10,6 +10,7 @@ import { ChatCompletionRequestMessage } from 'openai';
 import { modelToolMap } from '@/utils/plugin';
 import { gptMessage2ChatType } from '@/utils/adapt';
 import { addLog } from '../utils/tools';
+import { splitText2Chunks } from '@/utils/file';
 
 const reduceQueue = () => {
   global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -212,5 +213,16 @@ function formatSplitText(text: string) {
     }
   }
 
+  // empty result. direct split chunk
+  if (result.length === 0) {
+    const splitRes = splitText2Chunks({ text: text, maxLen: 500 });
+    splitRes.chunks.forEach((item) => {
+      result.push({
+        q: item,
+        a: ''
+      });
+    });
+  }
+
   return result;
 }

client/src/service/response.ts

@@ -44,7 +44,7 @@ export const jsonRes = <T = any>(
     if (typeof error === 'string') {
       msg = error;
     } else if (proxyError[error?.code]) {
-      msg = '接口连接异常';
+      msg = '网络连接异常';
     } else if (error?.response?.data?.error?.message) {
       msg = error?.response?.data?.error?.message;
     } else if (openaiAccountError[error?.response?.data?.error?.code]) {
@@ -85,7 +85,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => {
   if (typeof error === 'string') {
     msg = error;
   } else if (proxyError[error?.code]) {
-    msg = '接口连接异常';
+    msg = '网络连接异常';
   } else if (error?.response?.data?.error?.message) {
     msg = error?.response?.data?.error?.message;
   } else if (openaiAccountError[error?.response?.data?.error?.code]) {

client/src/utils/plugin/openai.ts

@@ -51,9 +51,18 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
   const adaptMessages = adaptChatItem_openAI({ messages, reserveId: true });
   const token = adaptMessages.reduce((sum, item) => {
     const text = `${item.role}\n${item.content}`;
-    const enc = getOpenAiEncMap();
+
-    const encodeText = enc.encode(text);
+    /* use textLen as tokens if encode error */
-    const tokens = encodeText.length + 3; // 补充估算值
+    const tokens = (() => {
+      try {
+        const enc = getOpenAiEncMap();
+        const encodeText = enc.encode(text);
+        return encodeText.length + 3; // 补充估算值
+      } catch (error) {
+        return text.length;
+      }
+    })();
+
     return sum + tokens;
   }, 0);
 
@@ -62,9 +71,14 @@ export function countOpenAIToken({ messages }: { messages: ChatItemType[] }) {
 
 export const openAiSliceTextByToken = ({ text, length }: { text: string; length: number }) => {
   const enc = getOpenAiEncMap();
-  const encodeText = enc.encode(text);
+
-  const decoder = new TextDecoder();
+  try {
-  return decoder.decode(enc.decode(encodeText.slice(0, length)));
+    const encodeText = enc.encode(text);
+    const decoder = new TextDecoder();
+    return decoder.decode(enc.decode(encodeText.slice(0, length)));
+  } catch (error) {
+    return text.slice(0, length);
+  }
 };
 
 export const authOpenAiKey = async (key: string) => {