feat: 修改模型数据可修改问题

feat: 模型数据导入

feat: redis 向量入库

feat: 向量索引

feat: 文件导入模型

perf: 交互

perf: prompt

.env.template

@@ -4,3 +4,5 @@ MONGODB_URI=
 MY_MAIL=
 MAILE_CODE=
 TOKEN_KEY=
+OPENAIKEY=
+REDIS_URL=

.prettierrc.js

@@ -18,4 +18,3 @@ module.exports = {
   htmlWhitespaceSensitivity: 'css',
   endOfLine: 'lf'
 };
-  

README.md

@@ -1,6 +1,6 @@
 # Fast GPT 
 
-Fast GPT 允许你是用自己的 openai API KEY 来快速的调用 openai 接口，包括 GPT3 及其微调方法，以及最新的 gpt3.5 接口。
+Fast GPT 允许你使用自己的 openai API KEY 来快速的调用 openai 接口，包括 GPT3 及其微调方法，以及最新的 gpt3.5 接口。
 
 ## 初始化
 复制 .env.template 成 .env.local ，填写核心参数  
@@ -11,7 +11,7 @@ AXIOS_PROXY_PORT=代理端口
 MONGODB_URI=mongo数据库地址（例如：mongodb://username:password@ip:27017/?authSource=admin&readPreference=primary&appname=MongoDB%20Compass&directConnection=true&ssl=false）
 MY_MAIL=发送验证码邮箱
 MAILE_CODE=邮箱秘钥（代理里设置的是QQ邮箱，不知道怎么找这个 code 的，可以百度搜"nodemailer发送邮件"）
-TOKEN_KEY=随便填一个，用于生成和校验token
+TOKEN_KEY=随便填一个，用于生成和校验 token
 ```
 
 ```bash
@@ -101,3 +101,12 @@ echo "Restart clash"
 
 #### 软件教程：Nginx
 ...没写，这个百度吧。
+
+#### redis
+
+```bash
+# 索引
+# FT.CREATE idx:model:data ON JSON PREFIX 1 model:data: SCHEMA $.modelId AS modelId TAG $.dataId AS dataId TAG $.vector AS vector VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT32
+# FT.CREATE idx:model:data:hash ON HASH PREFIX 1 model:data: SCHEMA modelId TAG dataId TAG vector VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT32
+FT.CREATE idx:model:data ON HASH PREFIX 1 model:data: SCHEMA modelId TAG userId TAG q TEXT text TEXT vector VECTOR FLAT 6 DIM 1536 DISTANCE_METRIC COSINE TYPE FLOAT32
+```

docs/FineTunes

@@ -1,4 +0,0 @@
-{"prompt": "sealos的介绍", "completion": "sealos 是以 kubernetes 为内核的云操作系统发行版"}
-{"prompt": "sealos是什么", "completion": "sealos 是以 kubernetes 为内核的云操作系统发行版"}
-{"prompt": "sealos安装的先决条件", "completion": "sealos 是一个简单的 go 二进制文件，可以安装在大多数 Linux 操作系统中。"}
-{"prompt": "sealos的CPU架构", "completion": "目前支持 amd64 和 arm64 架构。"}

package.json

@@ -41,6 +41,7 @@
     "react-hook-form": "^7.43.1",
     "react-markdown": "^8.0.5",
     "react-syntax-highlighter": "^15.5.0",
+    "redis": "^4.6.5",
     "rehype-katex": "^6.0.2",
     "remark-gfm": "^3.0.1",
     "remark-math": "^5.1.1",

pnpm-lock.yaml

@@ -47,6 +47,7 @@ specifiers:
   react-hook-form: ^7.43.1
   react-markdown: ^8.0.5
   react-syntax-highlighter: ^15.5.0
+  redis: ^4.6.5
   rehype-katex: ^6.0.2
   remark-gfm: ^3.0.1
   remark-math: ^5.1.1
@@ -87,6 +88,7 @@ dependencies:
   react-hook-form: registry.npmmirror.com/react-hook-form/7.43.1_react@18.2.0
   react-markdown: registry.npmmirror.com/react-markdown/8.0.5_pmekkgnqduwlme35zpnqhenc34
   react-syntax-highlighter: registry.npmmirror.com/react-syntax-highlighter/15.5.0_react@18.2.0
+  redis: registry.npmmirror.com/redis/4.6.5
   rehype-katex: registry.npmmirror.com/rehype-katex/6.0.2
   remark-gfm: registry.npmmirror.com/remark-gfm/3.0.1
   remark-math: registry.npmmirror.com/remark-math/5.1.1
@@ -4504,6 +4506,72 @@ packages:
     version: 2.11.6
     dev: false
 
+  registry.npmmirror.com/@redis/bloom/1.2.0_@redis+client@1.5.6:
+    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/bloom/-/bloom-1.2.0.tgz}
+    id: registry.npmmirror.com/@redis/bloom/1.2.0
+    name: '@redis/bloom'
+    version: 1.2.0
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
+    dev: false
+
+  registry.npmmirror.com/@redis/client/1.5.6:
+    resolution: {integrity: sha512-dFD1S6je+A47Lj22jN/upVU2fj4huR7S9APd7/ziUXsIXDL+11GPYti4Suv5y8FuXaN+0ZG4JF+y1houEJ7ToA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/client/-/client-1.5.6.tgz}
+    name: '@redis/client'
+    version: 1.5.6
+    engines: {node: '>=14'}
+    dependencies:
+      cluster-key-slot: registry.npmmirror.com/cluster-key-slot/1.1.2
+      generic-pool: registry.npmmirror.com/generic-pool/3.9.0
+      yallist: registry.npmmirror.com/yallist/4.0.0
+    dev: false
+
+  registry.npmmirror.com/@redis/graph/1.1.0_@redis+client@1.5.6:
+    resolution: {integrity: sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/graph/-/graph-1.1.0.tgz}
+    id: registry.npmmirror.com/@redis/graph/1.1.0
+    name: '@redis/graph'
+    version: 1.1.0
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
+    dev: false
+
+  registry.npmmirror.com/@redis/json/1.0.4_@redis+client@1.5.6:
+    resolution: {integrity: sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/json/-/json-1.0.4.tgz}
+    id: registry.npmmirror.com/@redis/json/1.0.4
+    name: '@redis/json'
+    version: 1.0.4
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
+    dev: false
+
+  registry.npmmirror.com/@redis/search/1.1.2_@redis+client@1.5.6:
+    resolution: {integrity: sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/search/-/search-1.1.2.tgz}
+    id: registry.npmmirror.com/@redis/search/1.1.2
+    name: '@redis/search'
+    version: 1.1.2
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
+    dev: false
+
+  registry.npmmirror.com/@redis/time-series/1.0.4_@redis+client@1.5.6:
+    resolution: {integrity: sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/time-series/-/time-series-1.0.4.tgz}
+    id: registry.npmmirror.com/@redis/time-series/1.0.4
+    name: '@redis/time-series'
+    version: 1.0.4
+    peerDependencies:
+      '@redis/client': ^1.0.0
+    dependencies:
+      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
+    dev: false
+
   registry.npmmirror.com/@rushstack/eslint-patch/1.2.0:
     resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@rushstack/eslint-patch/-/eslint-patch-1.2.0.tgz}
     name: '@rushstack/eslint-patch'
@@ -5562,6 +5630,13 @@ packages:
     version: 0.0.1
     dev: false
 
+  registry.npmmirror.com/cluster-key-slot/1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz}
+    name: cluster-key-slot
+    version: 1.1.2
+    engines: {node: '>=0.10.0'}
+    dev: false
+
   registry.npmmirror.com/color-convert/1.9.3:
     resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/color-convert/-/color-convert-1.9.3.tgz}
     name: color-convert
@@ -6799,6 +6874,13 @@ packages:
     version: 1.2.3
     dev: true
 
+  registry.npmmirror.com/generic-pool/3.9.0:
+    resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/generic-pool/-/generic-pool-3.9.0.tgz}
+    name: generic-pool
+    version: 3.9.0
+    engines: {node: '>= 4'}
+    dev: false
+
   registry.npmmirror.com/gensync/1.0.0-beta.2:
     resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/gensync/-/gensync-1.0.0-beta.2.tgz}
     name: gensync
@@ -9367,6 +9449,19 @@ packages:
       picomatch: registry.npmmirror.com/picomatch/2.3.1
     dev: false
 
+  registry.npmmirror.com/redis/4.6.5:
+    resolution: {integrity: sha512-O0OWA36gDQbswOdUuAhRL6mTZpHFN525HlgZgDaVNgCJIAZR3ya06NTESb0R+TUZ+BFaDpz6NnnVvoMx9meUFg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/redis/-/redis-4.6.5.tgz}
+    name: redis
+    version: 4.6.5
+    dependencies:
+      '@redis/bloom': registry.npmmirror.com/@redis/bloom/1.2.0_@redis+client@1.5.6
+      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
+      '@redis/graph': registry.npmmirror.com/@redis/graph/1.1.0_@redis+client@1.5.6
+      '@redis/json': registry.npmmirror.com/@redis/json/1.0.4_@redis+client@1.5.6
+      '@redis/search': registry.npmmirror.com/@redis/search/1.1.2_@redis+client@1.5.6
+      '@redis/time-series': registry.npmmirror.com/@redis/time-series/1.0.4_@redis+client@1.5.6
+    dev: false
+
   registry.npmmirror.com/refractor/3.6.0:
     resolution: {integrity: sha512-MY9W41IOWxxk31o+YvFCNyNzdkc9M20NoZK5vq6jkv4I/uh2zkWcfudj0Q1fovjUQJrNewS9NMzeTtqPf+n5EA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/refractor/-/refractor-3.6.0.tgz}
     name: refractor

src/api/chat.ts

@@ -39,6 +39,7 @@ export const postSaveChat = (data: { chatId: string; prompts: ChatItemType[] })
   POST('/chat/saveChat', data);
 
 /**
- * 删除最后一句
+ * 删除一句对话
  */
-export const delLastMessage = (chatId: string) => DELETE(`/chat/delLastMessage?chatId=${chatId}`);
+export const delChatRecordByIndex = (chatId: string, index: number) =>
+  DELETE(`/chat/delChatRecordByIndex?chatId=${chatId}&index=${index}`);

src/api/data.ts

@@ -3,12 +3,12 @@ import { RequestPaging } from '../types/index';
 import { Obj2Query } from '@/utils/tools';
 import type { DataListItem } from '@/types/data';
 import type { PagingData } from '../types/index';
-import { DataItemSchema } from '@/types/mongoSchema';
+import type { DataItemSchema } from '@/types/mongoSchema';
+import type { CreateDataProps } from '@/pages/data/components/CreateDataModal';
 
-export const getDataList = (data: RequestPaging) =>
+export const getDataList = () => GET<DataListItem[]>(`/data/getDataList`);
-  GET<PagingData<DataListItem>>(`/data/getDataList?${Obj2Query(data)}`);
 
-export const postData = (name: string) => POST<string>(`/data/postData?name=${name}`);
+export const postData = (data: CreateDataProps) => POST<string>(`/data/postData`, data);
 
 export const postSplitData = (dataId: string, text: string) =>
   POST(`/data/splitData`, { dataId, text });

src/api/model.ts

@@ -1,7 +1,9 @@
 import { GET, POST, DELETE, PUT } from './request';
-import type { ModelSchema } from '@/types/mongoSchema';
+import type { ModelSchema, ModelDataSchema, ModelSplitDataSchema } from '@/types/mongoSchema';
 import { ModelUpdateParams } from '@/types/model';
 import { TrainingItemType } from '../types/training';
+import { RequestPaging } from '../types/index';
+import { Obj2Query } from '@/utils/tools';
 
 export const getMyModels = () => GET<ModelSchema[]>('/model/list');
 
@@ -16,13 +18,46 @@ export const putModelById = (id: string, data: ModelUpdateParams) =>
   PUT(`/model/update?modelId=${id}`, data);
 
 export const postTrainModel = (id: string, form: FormData) =>
-  POST(`/model/train?modelId=${id}`, form, {
+  POST(`/model/train/train?modelId=${id}`, form, {
     headers: {
       'content-type': 'multipart/form-data'
     }
   });
 
-export const putModelTrainingStatus = (id: string) => PUT(`/model/putTrainStatus?modelId=${id}`);
+export const putModelTrainingStatus = (id: string) =>
+  PUT(`/model/train/putTrainStatus?modelId=${id}`);
 
 export const getModelTrainings = (id: string) =>
-  GET<TrainingItemType[]>(`/model/getTrainings?modelId=${id}`);
+  GET<TrainingItemType[]>(`/model/train/getTrainings?modelId=${id}`);
+
+/* 模型 data */
+
+type GetModelDataListProps = RequestPaging & {
+  modelId: string;
+};
+export const getModelDataList = (props: GetModelDataListProps) =>
+  GET(`/model/data/getModelData?${Obj2Query(props)}`);
+
+export const getExportDataList = (modelId: string) =>
+  GET<string>(`/model/data/exportModelData?modelId=${modelId}`);
+
+export const getModelSplitDataList = (modelId: string) =>
+  GET<ModelSplitDataSchema[]>(`/model/data/getSplitData?modelId=${modelId}`);
+
+export const postModelDataInput = (data: {
+  modelId: string;
+  data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[];
+}) => POST<number>(`/model/data/pushModelDataInput`, data);
+
+export const postModelDataFileText = (data: { modelId: string; text: string; prompt: string }) =>
+  POST(`/model/data/splitData`, data);
+
+export const postModelDataJsonData = (
+  modelId: string,
+  jsonData: { prompt: string; completion: string; vector?: number[] }[]
+) => POST(`/model/data/pushModelDataJson`, { modelId, data: jsonData });
+
+export const putModelDataById = (data: { dataId: string; text: string; q?: string }) =>
+  PUT('/model/data/putModelData', data);
+export const delOneModelData = (dataId: string) =>
+  DELETE(`/model/data/delModelDataById?dataId=${dataId}`);

src/api/response/chat.d.ts

@@ -7,8 +7,7 @@ export type InitChatResponse = {
   name: string;
   avatar: string;
   intro: string;
-  secret: ModelSchema.secret;
+  chatModel: ModelSchema.service.chatModel; // 对话模型名
-  chatModel: ModelSchema.service.ChatModel; // 模型名
+  modelName: ModelSchema.service.modelName; // 底层模型
   history: ChatItemType[];
-  isExpiredTime: boolean;
 };

src/components/Icon/icons/chatSend.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1679805359001" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="1328" xmlns:xlink="http://www.w3.org/1999/xlink" width="48" height="48"><path d="M416.583186 1022.194004c-5.417989 0-10.835979-1.203998-16.253968-3.611993-15.049971-6.621987-24.681952-21.069959-24.681952-37.323927l0-299.795414c0-12.641975 5.417989-24.079953 15.651969-31.905938 9.631981-7.825985 22.273956-10.23398 34.915932-7.825985l417.787184 99.931805 84.279835-599.590829c1.203998-9.631981-8.427984-16.253968-16.855967-11.437978L147.489712 573.102881l139.061728 35.517931c19.865961 4.815991 34.313933 22.875955 32.507937 43.343915-2.407995 25.885949-26.487948 42.139918-50.567901 36.119929L30.70194 627.282775c-16.253968-4.213992-27.691946-17.457966-30.099941-33.711934-2.407995-16.253968 5.417989-32.507937 19.865961-40.93592L962.59612 6.621987c13.243974-7.825985 30.099941-7.223986 43.343915 1.203998 12.641975 8.427984 19.865961 24.079953 17.457966 39.129924l-105.349794 750.090535c-1.805996 11.437978-7.825985 21.671958-17.457966 28.293945-9.631981 6.621987-21.069959 8.427984-32.507937 6.019988l-411.165197-98.125808 0 154.111699 81.87184-76.453851c15.049971-13.845973 37.925926-16.855967 54.179894-4.213992 20.46796 15.651969 21.069959 45.149912 3.009994 62.005879L444.275132 1011.358025C436.449148 1018.582011 426.817166 1022.194004 416.583186 1022.194004L416.583186 1022.194004z" p-id="1329"></path><path d="M416.583186 722.398589c-9.631981 0-19.263962-3.611993-27.089947-10.23398-16.855967-15.049971-18.059965-40.93592-3.009994-57.791887l216.117578-242.003527c15.049971-16.855967 40.93592-18.059965 57.791887-3.009994 16.855967 15.049971 18.059965 40.93592 3.009994 57.791887l-216.117578 242.003527C438.857143 718.184597 427.419165 722.398589 416.583186 722.398589L416.583186 722.398589z" p-id="1330"></path></svg>

src/components/Icon/icons/copy.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1679805221456" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="1173" xmlns:xlink="http://www.w3.org/1999/xlink" width="48" height="48"><path d="M267.3 834.6h-96.5c-27.4 0-49.7-22.3-49.7-49.7V115.2c0-27.4 22.3-49.7 49.7-49.7H727c27.4 0 49.7 22.3 49.7 49.7v96.5h-42.6v-96.5c0-3.9-3.2-7.1-7.1-7.1H170.8c-3.9 0-7.1 3.2-7.1 7.1v669.7c0 3.9 3.2 7.1 7.1 7.1h96.5v42.6z" p-id="1174"></path><path d="M851.9 959.5H295.7c-27.4 0-49.7-22.3-49.7-49.7V240.1c0-27.4 22.3-49.7 49.7-49.7h556.2c27.4 0 49.7 22.3 49.7 49.7v669.7c-0.1 27.4-22.3 49.7-49.7 49.7zM295.7 233c-3.9 0-7.1 3.2-7.1 7.1v669.7c0 3.9 3.2 7.1 7.1 7.1h556.2c3.9 0 7.1-3.2 7.1-7.1V240.1c0-3.9-3.2-7.1-7.1-7.1H295.7z" p-id="1175"></path></svg>

src/components/Icon/index.tsx

@@ -8,7 +8,9 @@ const map = {
   share: require('./icons/share.svg').default,
   home: require('./icons/home.svg').default,
   menu: require('./icons/menu.svg').default,
-  pay: require('./icons/pay.svg').default
+  pay: require('./icons/pay.svg').default,
+  copy: require('./icons/copy.svg').default,
+  chatSend: require('./icons/chatSend.svg').default
 };
 
 export type IconName = keyof typeof map;

src/components/Layout/auth.tsx

@@ -7,8 +7,7 @@ import { useQuery } from '@tanstack/react-query';
 
 const unAuthPage: { [key: string]: boolean } = {
   '/': true,
-  '/login': true,
+  '/login': true
-  '/chat': true
 };
 
 const Auth = ({ children }: { children: JSX.Element }) => {

src/components/Layout/index.tsx

@@ -1,5 +1,5 @@
 import React, { useEffect } from 'react';
-import { Box, useColorMode } from '@chakra-ui/react';
+import { Box, useColorMode, Flex } from '@chakra-ui/react';
 import Navbar from './navbar';
 import NavbarPhone from './navbarPhone';
 import { useRouter } from 'next/router';
@@ -26,12 +26,12 @@ const navbarList = [
     link: '/model/list',
     activeLink: ['/model/list', '/model/detail']
   },
-  {
+  // {
-    label: '数据',
+  //   label: '数据',
-    icon: 'icon-datafull',
+  //   icon: 'icon-datafull',
-    link: '/data/list',
+  //   link: '/data/list',
-    activeLink: ['/data/list', '/data/detail']
+  //   activeLink: ['/data/list', '/data/detail']
-  },
+  // },
   {
     label: '账号',
     icon: 'icon-yonghu-yuan',
@@ -62,29 +62,21 @@ const Layout = ({ children }: { children: JSX.Element }) => {
               <Box h={'100%'} position={'fixed'} left={0} top={0} w={'80px'}>
                 <Navbar navbarList={navbarList} />
               </Box>
-              <Box ml={'80px'} h={'100%'}>
+              <Box h={'100%'} ml={'80px'}>
-                <Box maxW={'1100px'} m={'auto'} h={'100%'} p={7} overflowY={'auto'}>
+                <Box h={'100%'} py={7} px={'5vw'} m={'auto'} overflowY={'auto'}>
                   <Auth>{children}</Auth>
                 </Box>
               </Box>
             </>
           ) : (
-            <Box pt={'60px'}>
+            <Flex h={'100%'} flexDirection={'column'}>
-              <Box
+              <Box h={'60px'} borderBottom={'1px solid rgba(0,0,0,0.1)'}>
-                h={'60px'}
-                position={'fixed'}
-                top={0}
-                left={0}
-                right={0}
-                zIndex={100}
-                borderBottom={'1px solid rgba(0,0,0,0.1)'}
-              >
                 <NavbarPhone navbarList={navbarList} />
               </Box>
-              <Box py={3} px={4}>
+              <Box flex={'1 0 0'} h={0} py={3} px={4} overflowY={'auto'}>
                 <Auth>{children}</Auth>
               </Box>
-            </Box>
+            </Flex>
           )}
         </Box>
       ) : (

src/components/Markdown/index.tsx

@@ -3,7 +3,7 @@ import ReactMarkdown from 'react-markdown';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { Box, Flex, useColorModeValue } from '@chakra-ui/react';
 import { useCopyData } from '@/utils/tools';
-import Icon from '@/components/Iconfont';
+import Icon from '@/components/Icon';
 import remarkGfm from 'remark-gfm';
 import remarkMath from 'remark-math';
 import rehypeKatex from 'rehype-katex';
@@ -41,7 +41,7 @@ const Markdown = ({ source, isChatting = false }: { source: string; isChatting?:
               >
                 <Box flex={1}>{match?.[1]}</Box>
                 <Flex cursor={'pointer'} onClick={() => copyData(code)} alignItems={'center'}>
-                  <Icon name={'icon-fuzhi'} width={15} height={15} color={'#fff'}></Icon>
+                  <Icon name={'copy'} width={15} height={15} fill={'#fff'}></Icon>
                   <Box ml={1}>复制代码</Box>
                 </Flex>
               </Flex>

src/components/ScrollData/index.tsx

@@ -53,7 +53,7 @@ const ScrollData = ({
   }, [elementRef, nextPage]);
 
   return (
-    <Box {...props} ref={elementRef} overflow={'auto'} position={'relative'}>
+    <Box {...props} ref={elementRef} overflowY={'auto'} position={'relative'}>
       {children}
       <Box
         mt={2}

src/components/WxConcat/index.tsx

@@ -23,7 +23,7 @@ const WxConcat = ({ onClose }: { onClose: () => void }) => {
         <ModalBody textAlign={'center'}>
           <Image
             style={{ margin: 'auto' }}
-            src={'/imgs/wxcode.jpg'}
+            src={'/imgs/wxcode300.jpg'}
             width={200}
             height={200}
             alt=""

src/constants/common.ts

@@ -10,6 +10,11 @@ export const introPage = `
 
 [Git 仓库](https://github.com/c121914yu/FastGPT)
 
+### 交流群/问题反馈
+wx: YNyiqi
+![](/imgs/wxcode300.jpg)
+
+
 ### 快速开始
 1. 使用邮箱注册账号。  
 2. 进入账号页面，添加关联账号，目前只有 openai 的账号可以添加，直接去 openai 官网，把 API Key 粘贴过来。  
@@ -17,44 +22,39 @@ export const introPage = `
 4. 进入模型页，创建一个模型，建议直接用 ChatGPT。    
 5. 在模型列表点击【对话】，即可使用 API 进行聊天。  
 
-### 模型配置
+### 定制 prompt
 
-1. **提示语**：会在每个对话框的第一句自动加入，用于限定该模型的对话内容。  
+1. 进入模型编辑页  
+2. 调整温度和提示词  
+3. 使用该模型对话。每次对话时，提示词和温度都会自动注入，方便管理个人的模型。建议把自己日常经常需要使用的 5~10 个方向预设好。
 
+### 知识库
 
-2. **单句最大长度**：每个聊天，单次输入内容的最大长度。  
+1. 创建模型时选择【知识库】  
+2. 进入模型编辑页  
+3. 导入数据，可以选择手动导入，或者选择文件导入。文件导入会自动调用 chatGPT 理解文件内容，并生成知识库。  
+4. 使用该模型对话。  
 
-
+注意：使用知识库模型对话时，tokens 消耗会加快。  
-3. **上下文最大长度**：每个聊天，最多的轮数除以2，建议设置为偶数。可以持续聊天，但是旧的聊天内容会被截断，AI 就不会知道被截取的内容。 
-例如：上下文最大长度为6。在第 4 轮对话时，第一轮对话的内容不会被计入。
-
-4. **过期时间**：生成对话框后，这个对话框多久过期。  
-
-5. **聊天最大加载次数**：单个对话框最多被加载几次，设置为-1代表不限制，正数代表只能加载 n 次，防止被盗刷。  
-
-### 对话框介绍
-
-1. 每个对话框以 chatId 作为标识。  
-2. 每次点击【对话】，都会生成新的对话框，无法回到旧的对话框。对话框内刷新，会恢复对话内容。  
-3. 直接分享对话框（网页）的链接给朋友，会共享同一个对话内容。但是！！！千万不要两个人同时用一个链接，会串味，还没解决这个问题。  
-4. 如果想分享一个纯的对话框，请点击侧边栏的分享按键。例如：  
-
-### 其他问题
-还有其他问题，可以加我 wx: YNyiqi，拉个交流群大家一起聊聊。
 `;
 
 export const chatProblem = `
+## 常见问题
+**内容长度**
+单次最长 4000 tokens, 上下文最长 8000 tokens, 上下文超长时会被截断。
+
+**删除和复制**
+点击对话头像，可以选择复制或删除该条内容。
+
 **代理出错**
 服务器代理不稳定，可以过一会儿再尝试。  
-
-**API key 问题**
-请把 openai 的 API key 粘贴到账号里再创建对话。如果是使用分享的对话，不需要填写 API key。  
 `;
 
 export const versionIntro = `
-## Fast GPT V2.0
+## Fast GPT V2.3
-* 优化记账模式： 不再根据文本长度进行记账，而是根据实际消耗 tokens 数量进行记账。
+* 数据集导出功能，可用于知识库分享。
-* 文本 QA 拆分: 可以在[数据]模块，使用 QA 拆分功能，粘贴文字或者选择文件均可以实现自动生成 QA。可以一键导出，用于微调模型。
+* 优化文件拆分功能，可自定义提示词。
+* 定制知识库：创建模型时可以选择【知识库】模型, 可以手动导入知识点或者直接导入一个文件自动学习。
 `;
 
 export const shareHint = `

src/constants/data.ts

@@ -0,0 +1,6 @@
+import type { DataType } from '@/types/data';
+
+export const DataTypeTextMap: Record<DataType, string> = {
+  QA: '问答拆分',
+  abstract: '摘要总结'
+};

src/constants/model.ts

@@ -1,18 +1,29 @@
-import type { ServiceName } from '@/types/mongoSchema';
+import type { ServiceName, ModelDataType, ModelSchema } from '@/types/mongoSchema';
-import { ModelSchema } from '../types/mongoSchema';
+import type { RedisModelDataItemType } from '@/types/redis';
 
 export enum ChatModelNameEnum {
   GPT35 = 'gpt-3.5-turbo',
-  GPT3 = 'text-davinci-003'
+  VECTOR_GPT = 'VECTOR_GPT',
+  GPT3 = 'text-davinci-003',
+  VECTOR = 'text-embedding-ada-002'
 }
 
+export const ChatModelNameMap = {
+  [ChatModelNameEnum.GPT35]: 'gpt-3.5-turbo',
+  [ChatModelNameEnum.VECTOR_GPT]: 'gpt-3.5-turbo',
+  [ChatModelNameEnum.GPT3]: 'text-davinci-003',
+  [ChatModelNameEnum.VECTOR]: 'text-embedding-ada-002'
+};
+
 export type ModelConstantsData = {
   serviceCompany: `${ServiceName}`;
   name: string;
   model: `${ChatModelNameEnum}`;
   trainName: string; // 空字符串代表不能训练
   maxToken: number;
+  contextMaxToken: number;
   maxTemperature: number;
+  trainedMaxToken: number; // 训练后最大多少tokens
   price: number; // 多少钱 / 1token，单位: 0.00001元
 };
 
@@ -23,18 +34,33 @@ export const modelList: ModelConstantsData[] = [
     model: ChatModelNameEnum.GPT35,
     trainName: '',
     maxToken: 4000,
+    contextMaxToken: 7500,
+    trainedMaxToken: 2000,
     maxTemperature: 2,
     price: 3
   },
   {
     serviceCompany: 'openai',
-    name: 'GPT3',
+    name: '知识库',
-    model: ChatModelNameEnum.GPT3,
+    model: ChatModelNameEnum.VECTOR_GPT,
-    trainName: 'davinci',
+    trainName: 'vector',
     maxToken: 4000,
-    maxTemperature: 2,
+    contextMaxToken: 7500,
-    price: 30
+    trainedMaxToken: 2000,
+    maxTemperature: 1,
+    price: 3
   }
+  // {
+  //   serviceCompany: 'openai',
+  //   name: 'GPT3',
+  //   model: ChatModelNameEnum.GPT3,
+  //   trainName: 'davinci',
+  //   maxToken: 4000,
+  //   contextMaxToken: 7500,
+  //   trainedMaxToken: 2000,
+  //   maxTemperature: 2,
+  //   price: 30
+  // }
 ];
 
 export enum TrainingStatusEnum {
@@ -70,6 +96,11 @@ export const formatModelStatus = {
   }
 };
 
+export const ModelDataStatusMap: Record<RedisModelDataItemType['status'], string> = {
+  ready: '训练完成',
+  waiting: '训练中'
+};
+
 export const defaultModel: ModelSchema = {
   _id: '',
   userId: '',

src/constants/redis.ts

@@ -0,0 +1,6 @@
+export const VecModelDataPrefix = 'model:data';
+export const VecModelDataIdx = `idx:${VecModelDataPrefix}:hash`;
+export enum ModelDataStatusEnum {
+  ready = 'ready',
+  waiting = 'waiting'
+}

src/constants/user.ts

@@ -1,6 +1,9 @@
 export enum BillTypeEnum {
   chat = 'chat',
   splitData = 'splitData',
+  QA = 'QA',
+  abstract = 'abstract',
+  vector = 'vector',
   return = 'return'
 }
 export enum PageTypeEnum {
@@ -11,6 +14,9 @@ export enum PageTypeEnum {
 
 export const BillTypeMap: Record<`${BillTypeEnum}`, string> = {
   [BillTypeEnum.chat]: '对话',
-  [BillTypeEnum.splitData]: '文本拆分',
+  [BillTypeEnum.splitData]: 'QA拆分',
+  [BillTypeEnum.QA]: 'QA拆分',
+  [BillTypeEnum.abstract]: '摘要总结',
+  [BillTypeEnum.vector]: '索引生成',
   [BillTypeEnum.return]: '退款'
 };

src/hooks/usePagination.tsx

@@ -0,0 +1,81 @@
+import { useState, useCallback, useMemo } from 'react';
+import type { PagingData } from '../types/index';
+import { IconButton, Flex, Box } from '@chakra-ui/react';
+import { ArrowBackIcon, ArrowForwardIcon } from '@chakra-ui/icons';
+import { useQuery, useMutation } from '@tanstack/react-query';
+import { useToast } from './useToast';
+
+export const usePagination = <T = any,>({
+  api,
+  pageSize = 10,
+  params = {}
+}: {
+  api: (data: any) => any;
+  pageSize?: number;
+  params?: Record<string, any>;
+}) => {
+  const { toast } = useToast();
+  const [pageNum, setPageNum] = useState(1);
+  const [total, setTotal] = useState(0);
+  const [data, setData] = useState<T[]>([]);
+  const maxPage = useMemo(() => Math.ceil(total / pageSize), [pageSize, total]);
+
+  const { mutate, isLoading } = useMutation({
+    mutationFn: async (num: number = pageNum) => {
+      try {
+        const res: PagingData<T> = await api({
+          pageNum: num,
+          pageSize,
+          ...params
+        });
+        setPageNum(num);
+        setTotal(res.total);
+        setData(res.data);
+      } catch (error: any) {
+        toast({
+          title: error?.message || '获取数据异常',
+          status: 'error'
+        });
+        console.log(error);
+      }
+    }
+  });
+  useQuery(['init'], () => {
+    mutate(1);
+    return null;
+  });
+
+  const Pagination = useCallback(() => {
+    return (
+      <Flex alignItems={'center'} justifyContent={'end'}>
+        <IconButton
+          isDisabled={pageNum === 1}
+          icon={<ArrowBackIcon />}
+          aria-label={'left'}
+          size={'sm'}
+          onClick={() => mutate(pageNum - 1)}
+        />
+        <Box mx={2}>
+          {pageNum}/{maxPage}
+        </Box>
+        <IconButton
+          isDisabled={pageNum === maxPage}
+          icon={<ArrowForwardIcon />}
+          aria-label={'left'}
+          size={'sm'}
+          onClick={() => mutate(pageNum + 1)}
+        />
+      </Flex>
+    );
+  }, [maxPage, mutate, pageNum]);
+
+  return {
+    pageNum,
+    pageSize,
+    total,
+    data,
+    isLoading,
+    Pagination,
+    getData: mutate
+  };
+};

src/hooks/usePaging.ts

@@ -8,7 +8,7 @@ export const usePaging = <T = any>({
   pageSize = 10,
   params = {}
 }: {
-  api: (data: any) => Promise<PagingData<T>>;
+  api: (data: any) => any;
   pageSize?: number;
   params?: Record<string, any>;
 }) => {
@@ -30,7 +30,7 @@ export const usePaging = <T = any>({
       setRequesting(true);
 
       try {
-        const res = await api({
+        const res: PagingData<T> = await api({
           pageNum: num,
           pageSize,
           ...params
@@ -59,7 +59,10 @@ export const usePaging = <T = any>({
     [api, isLoadAll, pageSize, params, requesting, toast]
   );
 
-  const nextPage = useCallback(() => getData(pageNum + 1), [getData, pageNum]);
+  const nextPage = useCallback(() => {
+    if (requesting || isLoadAll) return;
+    getData(pageNum + 1);
+  }, [getData, isLoadAll, pageNum, requesting]);
 
   useQuery(['init'], () => getData(1, true));
 
@@ -72,6 +75,7 @@ export const usePaging = <T = any>({
     requesting,
     isLoadAll,
     nextPage,
-    initRequesting
+    initRequesting,
+    setData
   };
 };

src/pages/_app.tsx

@@ -1,4 +1,5 @@
-import type { AppProps, NextWebVitalsMetric } from 'next/app';
+import { useEffect } from 'react';
+import type { AppProps } from 'next/app';
 import Script from 'next/script';
 import Head from 'next/head';
 import { ChakraProvider, ColorModeScript } from '@chakra-ui/react';
@@ -9,6 +10,7 @@ import NProgress from 'nprogress'; //nprogress module
 import Router from 'next/router';
 import 'nprogress/nprogress.css';
 import '../styles/reset.scss';
+import { useToast } from '@/hooks/useToast';
 
 //Binding events.
 Router.events.on('routeChangeStart', () => NProgress.start());
@@ -27,6 +29,17 @@ const queryClient = new QueryClient({
 });
 
 export default function App({ Component, pageProps }: AppProps) {
+  const { toast } = useToast();
+  // 校验是否支持 click 事件
+  useEffect(() => {
+    if (typeof document.createElement('div').click !== 'function') {
+      toast({
+        title: '你的浏览器版本过低',
+        status: 'warning'
+      });
+    }
+  }, [toast]);
+
   return (
     <>
       <Head>

src/pages/api/chat/chatGpt.ts

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { createParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser';
 import { connectToDatabase } from '@/service/mongo';
 import { getOpenAIApi, authChat } from '@/service/utils/chat';
-import { httpsAgent } from '@/service/utils/tools';
+import { httpsAgent, openaiChatFilter } from '@/service/utils/tools';
 import { ChatCompletionRequestMessage, ChatCompletionRequestMessageRoleEnum } from 'openai';
 import { ChatItemType } from '@/types/chat';
 import { jsonRes } from '@/service/response';
@@ -32,26 +32,38 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       prompt: ChatItemType;
       chatId: string;
     };
+
     const { authorization } = req.headers;
     if (!chatId || !prompt) {
       throw new Error('缺少参数');
     }
 
     await connectToDatabase();
+    let startTime = Date.now();
 
     const { chat, userApiKey, systemKey, userId } = await authChat(chatId, authorization);
 
     const model: ModelSchema = chat.modelId;
+    const modelConstantsData = modelList.find((item) => item.model === model.service.modelName);
+    if (!modelConstantsData) {
+      throw new Error('模型加载异常');
+    }
 
     // 读取对话内容
     const prompts = [...chat.content, prompt];
 
-    // 上下文长度过滤
+    // 如果有系统提示词，自动插入
-    const maxContext = model.security.contextMaxLen;
+    if (model.systemPrompt) {
-    const filterPrompts =
+      prompts.unshift({
-      prompts.length > maxContext ? prompts.slice(prompts.length - maxContext) : prompts;
+        obj: 'SYSTEM',
+        value: model.systemPrompt
+      });
+    }
 
-    // 格式化文本内容
+    // 控制在 tokens 数量，防止超出
+    const filterPrompts = openaiChatFilter(prompts, modelConstantsData.contextMaxToken);
+
+    // 格式化文本内容成 chatgpt 格式
     const map = {
       Human: ChatCompletionRequestMessageRoleEnum.User,
       AI: ChatCompletionRequestMessageRoleEnum.Assistant,
@@ -63,25 +75,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         content: item.value
       })
     );
-
+    // console.log(formatPrompts);
-    // 如果有系统提示词，自动插入
-    if (model.systemPrompt) {
-      formatPrompts.unshift({
-        role: 'system',
-        content: model.systemPrompt
-      });
-    }
-
     // 计算温度
-    const modelConstantsData = modelList.find((item) => item.model === model.service.modelName);
-    if (!modelConstantsData) {
-      throw new Error('模型异常');
-    }
     const temperature = modelConstantsData.maxTemperature * (model.temperature / 10);
 
     // 获取 chatAPI
     const chatAPI = getOpenAIApi(userApiKey || systemKey);
-    let startTime = Date.now();
     // 发出请求
     const chatResponse = await chatAPI.createChatCompletion(
       {
@@ -92,7 +91,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         frequency_penalty: 0.5, // 越大，重复内容越少
         presence_penalty: -0.5, // 越大，越容易出现新内容
         stream: true,
-        stop: ['。！？.!.']
+        stop: ['.!?。']
       },
       {
         timeout: 40000,
@@ -149,8 +148,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
     const promptsContent = formatPrompts.map((item) => item.content).join('');
     // 只有使用平台的 key 才计费
-    !userApiKey &&
     pushChatBill({
+      isPay: !userApiKey,
       modelName: model.service.modelName,
       userId,
       chatId,

src/pages/api/chat/delChatRecordByIndex.ts

@@ -0,0 +1,62 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Chat } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { chatId, index } = req.query as { chatId: string; index: string };
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+    if (!chatId || !index) {
+      throw new Error('缺少参数');
+    }
+
+    await connectToDatabase();
+
+    // 凭证校验
+    const userId = await authToken(authorization);
+
+    const chatRecord = await Chat.findById(chatId);
+
+    if (!chatRecord) {
+      throw new Error('找不到对话');
+    }
+
+    // 重新计算 index，跳过已经被删除的内容
+    let unDeleteIndex = +index;
+    let deletedIndex = 0;
+    for (deletedIndex = 0; deletedIndex < chatRecord.content.length; deletedIndex++) {
+      if (!chatRecord.content[deletedIndex].deleted) {
+        unDeleteIndex--;
+        if (unDeleteIndex < 0) {
+          break;
+        }
+      }
+    }
+
+    // 删除最一条数据库记录, 也就是预发送的那一条
+    await Chat.updateOne(
+      {
+        _id: chatId,
+        userId
+      },
+      {
+        $set: {
+          [`content.${deletedIndex}.deleted`]: true,
+          updateTime: Date.now()
+        }
+      }
+    );
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/chat/delLastMessage.ts

@@ -1,28 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, Chat } from '@/service/mongo';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    const { chatId } = req.query as { chatId: string };
-
-    if (!chatId) {
-      throw new Error('缺少参数');
-    }
-
-    await connectToDatabase();
-
-    // 删除最一条数据库记录, 也就是预发送的那一条
-    await Chat.findByIdAndUpdate(chatId, {
-      $pop: { content: 1 },
-      updateTime: Date.now()
-    });
-
-    jsonRes(res);
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}

src/pages/api/chat/generate.ts

@@ -14,7 +14,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     const { authorization } = req.headers;
 
     if (!authorization) {
-      throw new Error('无权操作');
+      throw new Error('无权生成对话');
     }
 
     if (!modelId) {
@@ -42,7 +42,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       modelId,
       expiredTime: Date.now() + model.security.expiredTime,
       loadAmount: model.security.maxLoadAmount,
-      updateTime: Date.now(),
       isShare: isShare === 'true',
       content: []
     });

src/pages/api/chat/gpt3.ts

@@ -51,11 +51,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       prompts.length > maxContext ? prompts.slice(prompts.length - maxContext) : prompts;
 
     // 格式化文本内容
-    const map = {
-      Human: 'Human',
-      AI: 'AI',
-      SYSTEM: 'SYSTEM'
-    };
     const formatPrompts: string[] = filterPrompts.map((item: ChatItemType) => item.value);
     // 如果有系统提示词，自动插入
     if (model.systemPrompt) {
@@ -67,14 +62,24 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     // 计算温度
     const modelConstantsData = modelList.find((item) => item.model === model.service.modelName);
     if (!modelConstantsData) {
-      throw new Error('模型异常');
+      throw new Error('模型异常,请用 chatgpt 模型');
     }
     const temperature = modelConstantsData.maxTemperature * (model.temperature / 10);
 
     // 获取 chatAPI
     const chatAPI = getOpenAIApi(userApiKey || systemKey);
     let startTime = Date.now();
-
+    // console.log({
+    //   model: model.service.chatModel,
+    //   temperature: temperature,
+    //   prompt: promptText,
+    //   stream: true,
+    //   max_tokens:
+    //     model.trainingTimes > 0 ? modelConstantsData.trainedMaxToken : modelConstantsData.maxToken,
+    //   presence_penalty: -0.5, // 越大，越容易出现新内容
+    //   frequency_penalty: 0.5, // 越大，重复内容越少
+    //   stop: [`###`]
+    // });
     // 发出请求
     const chatResponse = await chatAPI.createCompletion(
       {
@@ -82,10 +87,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         temperature: temperature,
         prompt: promptText,
         stream: true,
-        max_tokens: modelConstantsData.maxToken,
+        max_tokens:
-        presence_penalty: 0, // 越大，越容易出现新内容
+          model.trainingTimes > 0
-        frequency_penalty: 0, // 越大，重复内容越少
+            ? modelConstantsData.trainedMaxToken
-        stop: ['。！？.!.', `</s>`]
+            : modelConstantsData.maxToken,
+        presence_penalty: -0.5, // 越大，越容易出现新内容
+        frequency_penalty: 0.5, // 越大，重复内容越少
+        stop: [`###`, '。！？.!.']
       },
       {
         timeout: 40000,
@@ -113,10 +121,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       try {
         const json = JSON.parse(data);
         const content: string = json?.choices?.[0].text || '';
+        // console.log('content:', content);
         if (!content || (responseContent === '' && content === '\n')) return;
 
         responseContent += content;
-        // console.log('content:', content);
         !stream.destroyed && stream.push(content.replace(/\n/g, '<br/>'));
       } catch (error) {
         error;
@@ -141,8 +149,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     stream.destroy();
 
     // 只有使用平台的 key 才计费
-    !userApiKey &&
     pushChatBill({
+      isPay: !userApiKey,
       modelName: model.service.modelName,
       userId,
       chatId,

src/pages/api/chat/init.ts

@@ -3,10 +3,14 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat } from '@/service/mongo';
 import type { ChatPopulate } from '@/types/mongoSchema';
 import type { InitChatResponse } from '@/api/response/chat';
+import { authToken } from '@/service/utils/tools';
 
-/* 获取我的模型 */
+/* 初始化我的聊天框，需要身份验证 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    const { authorization } = req.headers;
+    const userId = await authToken(authorization);
+
     const { chatId } = req.query as { chatId: string };
 
     if (!chatId) {
@@ -16,7 +20,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     await connectToDatabase();
 
     // 获取 chat 数据
-    const chat = await Chat.findById<ChatPopulate>(chatId).populate({
+    const chat = await Chat.findOne<ChatPopulate>({
+      _id: chatId,
+      userId
+    }).populate({
       path: 'modelId',
       options: {
         strictPopulate: false
@@ -27,28 +34,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       throw new Error('聊天框不存在');
     }
 
-    if (chat.loadAmount > 0) {
+    // filter 掉被 deleted 的内容
-      await Chat.updateOne(
+    chat.content = chat.content.filter((item) => item.deleted !== true);
-        {
-          _id: chat._id
-        },
-        {
-          $inc: { loadAmount: -1 }
-        }
-      );
-    }
 
     const model = chat.modelId;
     jsonRes<InitChatResponse>(res, {
-      code: 201,
       data: {
         chatId: chat._id,
-        isExpiredTime: chat.loadAmount === 0 || chat.expiredTime <= Date.now(),
         modelId: model._id,
         name: model.name,
         avatar: model.avatar,
         intro: model.intro,
-        secret: model.security,
+        modelName: model.service.modelName,
         chatModel: model.service.chatModel,
         history: chat.content
       }

src/pages/api/chat/lafGpt.ts

@@ -0,0 +1,277 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { createParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser';
+import { connectToDatabase } from '@/service/mongo';
+import { getOpenAIApi, authChat } from '@/service/utils/chat';
+import { httpsAgent, openaiChatFilter, systemPromptFilter } from '@/service/utils/tools';
+import { ChatCompletionRequestMessage, ChatCompletionRequestMessageRoleEnum } from 'openai';
+import { ChatItemType } from '@/types/chat';
+import { jsonRes } from '@/service/response';
+import type { ModelSchema } from '@/types/mongoSchema';
+import { PassThrough } from 'stream';
+import { modelList } from '@/constants/model';
+import { pushChatBill } from '@/service/events/pushBill';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataPrefix } from '@/constants/redis';
+import { vectorToBuffer } from '@/utils/tools';
+import { openaiCreateEmbedding } from '@/service/utils/openai';
+
+/* 发送提示词 */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  let step = 0; // step=1时，表示开始了流响应
+  const stream = new PassThrough();
+  stream.on('error', () => {
+    console.log('error: ', 'stream error');
+    stream.destroy();
+  });
+  res.on('close', () => {
+    stream.destroy();
+  });
+  res.on('error', () => {
+    console.log('error: ', 'request error');
+    stream.destroy();
+  });
+
+  try {
+    const { chatId, prompt } = req.body as {
+      prompt: ChatItemType;
+      chatId: string;
+    };
+
+    const { authorization } = req.headers;
+    if (!chatId || !prompt) {
+      throw new Error('缺少参数');
+    }
+
+    await connectToDatabase();
+    const redis = await connectRedis();
+    let startTime = Date.now();
+
+    const { chat, userApiKey, systemKey, userId } = await authChat(chatId, authorization);
+
+    const model: ModelSchema = chat.modelId;
+    const modelConstantsData = modelList.find((item) => item.model === model.service.modelName);
+    if (!modelConstantsData) {
+      throw new Error('模型加载异常');
+    }
+    // 获取 chatAPI
+    const chatAPI = getOpenAIApi(userApiKey || systemKey);
+
+    // 请求一次 chatgpt 拆解需求
+    const promptResponse = await chatAPI.createChatCompletion(
+      {
+        model: model.service.chatModel,
+        temperature: 0,
+        // max_tokens: modelConstantsData.maxToken,
+        messages: [
+          {
+            role: 'system',
+            content: `服务端逻辑生成器。根据用户输入的需求，拆解成代码实现的步骤，并按格式返回： 1.\n2.\n3.\n ......
+          
+          下面是一些例子：
+          实现一个手机号注册账号的方法，包含两个函数
+          * 发送手机验证码函数： 
+          1. 从 query 中获取 phone 
+          2. 校验手机号格式是否正确，不正确返回{error: "手机号格式错误"} 
+          3. 给 phone 发送一个短信验证码，验证码长度为6位字符串，内容为：你正在注册laf, 验证码为：code 
+          4. 数据库添加数据，表为"codes"，内容为 {phone, code} 
+          * 注册函数 
+          1. 从 body 中获取 phone 和 code 
+          2. 校验手机号格式是否正确，不正确返回{error: "手机号格式错误"} 
+          2. 获取数据库数据，表为"codes"，查找是否有符合 phone, code 等于body参数的记录，没有的话返回 {error:"验证码不正确"} 
+          4. 添加数据库数据，表为"users" ，内容为{phone, code, createTime}
+          5. 删除数据库数据，删除 code 记录
+          ---------------
+          更新博客记录。传入blogId，blogText，tags，还需要记录更新的时间
+          1. 从 body 中获取 blogId，blogText 和 tags
+          2. 校验 blogId 是否为空，为空则返回 {error: "博客ID不能为空"}
+          3. 校验 blogText 是否为空，为空则返回 {error: "博客内容不能为空"}
+          4. 校验 tags 是否为数组，不是则返回 {error: "标签必须为数组"}
+          5. 获取当前时间，记录为 updateTime
+          6. 更新数据库数据，表为"blogs"，更新符合 blogId 的记录的内容为{blogText, tags, updateTime}
+          7. 返回结果 {message: "更新博客记录成功"}`
+          },
+          {
+            role: 'user',
+            content: prompt.value
+          }
+        ]
+      },
+      {
+        timeout: 40000,
+        httpsAgent
+      }
+    );
+
+    const promptResolve = promptResponse.data.choices?.[0]?.message?.content || '';
+    if (!promptResolve) {
+      throw new Error('gpt 异常');
+    }
+
+    prompt.value += `\n${promptResolve}`;
+    console.log('prompt resolve success, time:', `${(Date.now() - startTime) / 1000}s`);
+
+    // 获取提示词的向量
+    const { vector: promptVector } = await openaiCreateEmbedding({
+      isPay: !userApiKey,
+      apiKey: userApiKey || systemKey,
+      userId,
+      text: prompt.value
+    });
+
+    // 读取对话内容
+    const prompts = [...chat.content, prompt];
+
+    // 搜索系统提示词, 按相似度从 redis 中搜出相关的 q 和 text
+    const redisData: any[] = await redis.sendCommand([
+      'FT.SEARCH',
+      `idx:${VecModelDataPrefix}:hash`,
+      `@modelId:{${String(
+        chat.modelId._id
+      )}} @vector:[VECTOR_RANGE 0.25 $blob]=>{$YIELD_DISTANCE_AS: score}`,
+      // `@modelId:{${String(chat.modelId._id)}}=>[KNN 10 @vector $blob AS score]`,
+      'RETURN',
+      '1',
+      'text',
+      'SORTBY',
+      'score',
+      'PARAMS',
+      '2',
+      'blob',
+      vectorToBuffer(promptVector),
+      'LIMIT',
+      '0',
+      '20',
+      'DIALECT',
+      '2'
+    ]);
+
+    // 格式化响应值，获取 qa
+    const formatRedisPrompt = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
+      .map((i) => {
+        if (!redisData[i]) return '';
+        const text = (redisData[i][1] as string) || '';
+
+        if (!text) return '';
+
+        return text;
+      })
+      .filter((item) => item);
+
+    if (formatRedisPrompt.length === 0) {
+      throw new Error('对不起，我没有找到你的问题');
+    }
+
+    // textArr 筛选，最多 3000 tokens
+    const systemPrompt = systemPromptFilter(formatRedisPrompt, 3400);
+
+    prompts.unshift({
+      obj: 'SYSTEM',
+      value: `${model.systemPrompt} 知识库内容是最新的，知识库内容为: "${systemPrompt}"`
+    });
+
+    // 控制在 tokens 数量，防止超出
+    const filterPrompts = openaiChatFilter(prompts, modelConstantsData.contextMaxToken);
+
+    // 格式化文本内容成 chatgpt 格式
+    const map = {
+      Human: ChatCompletionRequestMessageRoleEnum.User,
+      AI: ChatCompletionRequestMessageRoleEnum.Assistant,
+      SYSTEM: ChatCompletionRequestMessageRoleEnum.System
+    };
+    const formatPrompts: ChatCompletionRequestMessage[] = filterPrompts.map(
+      (item: ChatItemType) => ({
+        role: map[item.obj],
+        content: item.value
+      })
+    );
+    console.log(formatPrompts);
+    // 计算温度
+    const temperature = modelConstantsData.maxTemperature * (model.temperature / 10);
+
+    // 发出请求
+    const chatResponse = await chatAPI.createChatCompletion(
+      {
+        model: model.service.chatModel,
+        temperature: temperature,
+        // max_tokens: modelConstantsData.maxToken,
+        messages: formatPrompts,
+        frequency_penalty: 0.5, // 越大，重复内容越少
+        presence_penalty: -0.5, // 越大，越容易出现新内容
+        stream: true
+      },
+      {
+        timeout: 40000,
+        responseType: 'stream',
+        httpsAgent
+      }
+    );
+
+    console.log('api response time:', `${(Date.now() - startTime) / 1000}s`);
+
+    // 创建响应流
+    res.setHeader('Content-Type', 'text/event-stream;charset-utf-8');
+    res.setHeader('Access-Control-Allow-Origin', '*');
+    res.setHeader('X-Accel-Buffering', 'no');
+    res.setHeader('Cache-Control', 'no-cache, no-transform');
+    step = 1;
+
+    let responseContent = '';
+    stream.pipe(res);
+
+    const onParse = async (event: ParsedEvent | ReconnectInterval) => {
+      if (event.type !== 'event') return;
+      const data = event.data;
+      if (data === '[DONE]') return;
+      try {
+        const json = JSON.parse(data);
+        const content: string = json?.choices?.[0].delta.content || '';
+        if (!content || (responseContent === '' && content === '\n')) return;
+
+        responseContent += content;
+        // console.log('content:', content)
+        !stream.destroyed && stream.push(content.replace(/\n/g, '<br/>'));
+      } catch (error) {
+        error;
+      }
+    };
+
+    const decoder = new TextDecoder();
+    try {
+      for await (const chunk of chatResponse.data as any) {
+        if (stream.destroyed) {
+          // 流被中断了，直接忽略后面的内容
+          break;
+        }
+        const parser = createParser(onParse);
+        parser.feed(decoder.decode(chunk));
+      }
+    } catch (error) {
+      console.log('pipe error', error);
+    }
+    // close stream
+    !stream.destroyed && stream.push(null);
+    stream.destroy();
+
+    const promptsContent = formatPrompts.map((item) => item.content).join('');
+    // 只有使用平台的 key 才计费
+    pushChatBill({
+      isPay: !userApiKey,
+      modelName: model.service.modelName,
+      userId,
+      chatId,
+      text: promptsContent + responseContent
+    });
+  } catch (err: any) {
+    if (step === 1) {
+      // 直接结束流
+      console.log('error，结束');
+      stream.destroy();
+    } else {
+      res.status(500);
+      jsonRes(res, {
+        code: 500,
+        error: err
+      });
+    }
+  }
+}

src/pages/api/chat/saveChat.ts

@@ -27,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
           }))
         }
       },
-      updateTime: Date.now()
+      updateTime: new Date()
     });
 
     jsonRes(res);

src/pages/api/chat/vectorGpt.ts

@@ -0,0 +1,222 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { createParser, ParsedEvent, ReconnectInterval } from 'eventsource-parser';
+import { connectToDatabase } from '@/service/mongo';
+import { getOpenAIApi, authChat } from '@/service/utils/chat';
+import { httpsAgent, openaiChatFilter, systemPromptFilter } from '@/service/utils/tools';
+import { ChatCompletionRequestMessage, ChatCompletionRequestMessageRoleEnum } from 'openai';
+import { ChatItemType } from '@/types/chat';
+import { jsonRes } from '@/service/response';
+import type { ModelSchema } from '@/types/mongoSchema';
+import { PassThrough } from 'stream';
+import { modelList } from '@/constants/model';
+import { pushChatBill } from '@/service/events/pushBill';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataPrefix } from '@/constants/redis';
+import { vectorToBuffer } from '@/utils/tools';
+import { openaiCreateEmbedding } from '@/service/utils/openai';
+
+/* 发送提示词 */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  let step = 0; // step=1时，表示开始了流响应
+  const stream = new PassThrough();
+  stream.on('error', () => {
+    console.log('error: ', 'stream error');
+    stream.destroy();
+  });
+  res.on('close', () => {
+    stream.destroy();
+  });
+  res.on('error', () => {
+    console.log('error: ', 'request error');
+    stream.destroy();
+  });
+
+  try {
+    const { chatId, prompt } = req.body as {
+      prompt: ChatItemType;
+      chatId: string;
+    };
+
+    const { authorization } = req.headers;
+    if (!chatId || !prompt) {
+      throw new Error('缺少参数');
+    }
+
+    await connectToDatabase();
+    const redis = await connectRedis();
+    let startTime = Date.now();
+
+    const { chat, userApiKey, systemKey, userId } = await authChat(chatId, authorization);
+
+    const model: ModelSchema = chat.modelId;
+    const modelConstantsData = modelList.find((item) => item.model === model.service.modelName);
+    if (!modelConstantsData) {
+      throw new Error('模型加载异常');
+    }
+
+    // 读取对话内容
+    const prompts = [...chat.content, prompt];
+
+    // 获取提示词的向量
+    const { vector: promptVector, chatAPI } = await openaiCreateEmbedding({
+      isPay: !userApiKey,
+      apiKey: userApiKey || systemKey,
+      userId,
+      text: prompt.value
+    });
+
+    // 搜索系统提示词, 按相似度从 redis 中搜出相关的 q 和 text
+    const redisData: any[] = await redis.sendCommand([
+      'FT.SEARCH',
+      `idx:${VecModelDataPrefix}:hash`,
+      `@modelId:{${String(
+        chat.modelId._id
+      )}} @vector:[VECTOR_RANGE 0.25 $blob]=>{$YIELD_DISTANCE_AS: score}`,
+      // `@modelId:{${String(chat.modelId._id)}}=>[KNN 10 @vector $blob AS score]`,
+      'RETURN',
+      '1',
+      'text',
+      'SORTBY',
+      'score',
+      'PARAMS',
+      '2',
+      'blob',
+      vectorToBuffer(promptVector),
+      'LIMIT',
+      '0',
+      '20',
+      'DIALECT',
+      '2'
+    ]);
+
+    // 格式化响应值，获取 qa
+    const formatRedisPrompt = [2, 4, 6, 8, 10, 12, 14, 16, 18, 20]
+      .map((i) => {
+        if (!redisData[i]) return '';
+        const text = (redisData[i][1] as string) || '';
+
+        if (!text) return '';
+
+        return text;
+      })
+      .filter((item) => item);
+
+    if (formatRedisPrompt.length === 0) {
+      throw new Error('对不起，我没有找到你的问题');
+    }
+
+    // textArr 筛选，最多 3000 tokens
+    const systemPrompt = systemPromptFilter(formatRedisPrompt, 3400);
+
+    prompts.unshift({
+      obj: 'SYSTEM',
+      value: `${model.systemPrompt} 知识库内容是最新的，知识库内容为: "${systemPrompt}"`
+    });
+
+    // 控制在 tokens 数量，防止超出
+    const filterPrompts = openaiChatFilter(prompts, modelConstantsData.contextMaxToken);
+
+    // 格式化文本内容成 chatgpt 格式
+    const map = {
+      Human: ChatCompletionRequestMessageRoleEnum.User,
+      AI: ChatCompletionRequestMessageRoleEnum.Assistant,
+      SYSTEM: ChatCompletionRequestMessageRoleEnum.System
+    };
+    const formatPrompts: ChatCompletionRequestMessage[] = filterPrompts.map(
+      (item: ChatItemType) => ({
+        role: map[item.obj],
+        content: item.value
+      })
+    );
+    // console.log(formatPrompts);
+    // 计算温度
+    const temperature = modelConstantsData.maxTemperature * (model.temperature / 10);
+
+    // 发出请求
+    const chatResponse = await chatAPI.createChatCompletion(
+      {
+        model: model.service.chatModel,
+        temperature: temperature,
+        // max_tokens: modelConstantsData.maxToken,
+        messages: formatPrompts,
+        frequency_penalty: 0.5, // 越大，重复内容越少
+        presence_penalty: -0.5, // 越大，越容易出现新内容
+        stream: true
+      },
+      {
+        timeout: 40000,
+        responseType: 'stream',
+        httpsAgent
+      }
+    );
+
+    console.log('api response time:', `${(Date.now() - startTime) / 1000}s`);
+
+    // 创建响应流
+    res.setHeader('Content-Type', 'text/event-stream;charset-utf-8');
+    res.setHeader('Access-Control-Allow-Origin', '*');
+    res.setHeader('X-Accel-Buffering', 'no');
+    res.setHeader('Cache-Control', 'no-cache, no-transform');
+    step = 1;
+
+    let responseContent = '';
+    stream.pipe(res);
+
+    const onParse = async (event: ParsedEvent | ReconnectInterval) => {
+      if (event.type !== 'event') return;
+      const data = event.data;
+      if (data === '[DONE]') return;
+      try {
+        const json = JSON.parse(data);
+        const content: string = json?.choices?.[0].delta.content || '';
+        if (!content || (responseContent === '' && content === '\n')) return;
+
+        responseContent += content;
+        // console.log('content:', content)
+        !stream.destroyed && stream.push(content.replace(/\n/g, '<br/>'));
+      } catch (error) {
+        error;
+      }
+    };
+
+    const decoder = new TextDecoder();
+    try {
+      for await (const chunk of chatResponse.data as any) {
+        if (stream.destroyed) {
+          // 流被中断了，直接忽略后面的内容
+          break;
+        }
+        const parser = createParser(onParse);
+        parser.feed(decoder.decode(chunk));
+      }
+    } catch (error) {
+      console.log('pipe error', error);
+    }
+    // close stream
+    !stream.destroyed && stream.push(null);
+    stream.destroy();
+
+    const promptsContent = formatPrompts.map((item) => item.content).join('');
+    // 只有使用平台的 key 才计费
+    pushChatBill({
+      isPay: !userApiKey,
+      modelName: model.service.modelName,
+      userId,
+      chatId,
+      text: promptsContent + responseContent
+    });
+    // jsonRes(res);
+  } catch (err: any) {
+    if (step === 1) {
+      // 直接结束流
+      console.log('error，结束');
+      stream.destroy();
+    } else {
+      res.status(500);
+      jsonRes(res, {
+        code: 500,
+        error: err
+      });
+    }
+  }
+}

src/pages/api/data/getDataList.ts

@@ -3,18 +3,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Data, DataItem } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
-import type { DataSchema } from '@/types/mongoSchema';
 import type { DataListItem } from '@/types/data';
-import type { PagingData } from '@/types';
 import mongoose from 'mongoose';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     const { authorization } = req.headers;
-    let { pageNum = 1, pageSize = 10 } = req.query as { pageNum: string; pageSize: string };
-
-    pageNum = +pageNum;
-    pageSize = +pageSize;
 
     if (!authorization) {
       throw new Error('缺少登录凭证');
@@ -34,12 +28,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       {
         $sort: { createTime: -1 } // 按照创建时间倒序排列
       },
-      {
-        $skip: (pageNum - 1) * pageSize // 跳过前面的数据
-      },
-      {
-        $limit: pageSize // 取出指定数量的数据
-      },
       {
         $lookup: {
           from: 'dataitems',
@@ -71,13 +59,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       }
     ]);
 
-    jsonRes<PagingData<DataListItem>>(res, {
+    jsonRes(res, {
-      data: {
+      data: datalist
-        pageNum,
-        pageSize,
-        data: datalist,
-        total: 1
-      }
     });
   } catch (err) {
     jsonRes(res, {

src/pages/api/data/postData.ts

@@ -2,11 +2,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Data } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
+import type { DataType } from '@/types/data';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    let { name } = req.query as { name: string };
+    let { name, type } = req.body as { name: string; type: DataType };
-    if (!name) {
+    if (!name || !type) {
       throw new Error('参数错误');
     }
     await connectToDatabase();
@@ -18,7 +19,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     // 生成 data 集合
     const data = await Data.create({
       userId,
-      name
+      name,
+      type
     });
 
     jsonRes(res, {

src/pages/api/data/splitData.ts

@@ -1,42 +1,64 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, Data, DataItem } from '@/service/mongo';
+import { connectToDatabase, DataItem, Data } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
 import { generateQA } from '@/service/events/generateQA';
+import { generateAbstract } from '@/service/events/generateAbstract';
+import { encode } from 'gpt-token-utils';
 
+/* 拆分数据成QA */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    let { text, dataId } = req.body as { text: string; dataId: string };
+    const { text, dataId } = req.body as { text: string; dataId: string };
     if (!text || !dataId) {
       throw new Error('参数错误');
     }
-    text = text.replace(/\n+/g, '\n');
     await connectToDatabase();
 
     const { authorization } = req.headers;
 
     const userId = await authToken(authorization);
 
-    const dataItems: any[] = [];
+    const DataRecord = await Data.findById(dataId);
 
-    // 格式化文本长度
+    if (!DataRecord) {
-    for (let i = 0; i <= text.length / 1000; i++) {
+      throw new Error('找不到数据集');
+    }
+    const replaceText = text.replace(/[\\n]+/g, ' ');
+
+    // 文本拆分成 chunk
+    let chunks = replaceText.match(/[^!?.。]+[!?.。]/g) || [];
+
+    const dataItems: any[] = [];
+    let splitText = '';
+
+    chunks.forEach((chunk) => {
+      splitText += chunk;
+      const tokens = encode(splitText).length;
+      if (tokens >= 780) {
         dataItems.push({
-        temperature: 0,
           userId,
           dataId,
-        text: text.slice(i * 1000, (i + 1) * 1000),
+          type: DataRecord.type,
+          text: splitText,
           status: 1
         });
+        splitText = '';
       }
+    });
 
     // 批量插入数据
     await DataItem.insertMany(dataItems);
 
+    try {
       generateQA();
+      generateAbstract();
+    } catch (error) {
+      error;
+    }
 
     jsonRes(res, {
-      data: dataItems.length
+      data: { chunks, replaceText }
     });
   } catch (err) {
     jsonRes(res, {

src/pages/api/model/create.ts

@@ -3,7 +3,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
 import { authToken } from '@/service/utils/tools';
-import { ModelStatusEnum, modelList, ChatModelNameEnum } from '@/constants/model';
+import { ModelStatusEnum, modelList, ChatModelNameEnum, ChatModelNameMap } from '@/constants/model';
 import { Model } from '@/service/models/model';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -33,15 +33,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     await connectToDatabase();
 
-    // 重名校验
-    const authRepeatName = await Model.findOne({
-      name,
-      userId
-    });
-    if (authRepeatName) {
-      throw new Error('模型名重复');
-    }
-
     // 上限校验
     const authCount = await Model.countDocuments({
       userId
@@ -57,9 +48,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       status: ModelStatusEnum.running,
       service: {
         company: modelItem.serviceCompany,
-        trainId: modelItem.trainName,
+        trainId: '',
-        chatModel: modelItem.model,
+        chatModel: ChatModelNameMap[modelItem.model], // 聊天时用的模型
-        modelName: modelItem.model
+        modelName: modelItem.model // 最底层的模型，不会变，用于计费等核心操作
       }
     });
 

src/pages/api/model/data/delModelDataById.ts

@@ -0,0 +1,41 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { authToken } from '@/service/utils/tools';
+import { connectRedis } from '@/service/redis';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    let { dataId } = req.query as {
+      dataId: string;
+    };
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+
+    if (!dataId) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const userId = await authToken(authorization);
+
+    const redis = await connectRedis();
+
+    // 校验是否为该用户的数据
+    const dataItemUserId = await redis.hGet(dataId, 'userId');
+    if (dataItemUserId !== userId) {
+      throw new Error('无权操作');
+    }
+    // 删除
+    await redis.del(dataId);
+    jsonRes(res);
+  } catch (err) {
+    console.log(err);
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/exportModelData.ts

@@ -0,0 +1,69 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataIdx } from '@/constants/redis';
+import { BufferToVector } from '@/utils/tools';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    let { modelId } = req.query as {
+      modelId: string;
+    };
+
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+
+    if (!modelId) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const userId = await authToken(authorization);
+
+    await connectToDatabase();
+    const redis = await connectRedis();
+
+    // 从 redis 中获取数据
+    const searchRes = await redis.ft.search(
+      VecModelDataIdx,
+      `@modelId:{${modelId}} @userId:{${userId}}`,
+      {
+        RETURN: ['q', 'text', 'rawVector'],
+        LIMIT: {
+          from: 0,
+          size: 10000
+        }
+      }
+    );
+
+    const data = searchRes.documents
+      .filter((item) => {
+        if (!item?.value?.rawVector) return false;
+        try {
+          JSON.parse(item.value.rawVector as string);
+          return true;
+        } catch (error) {
+          return false;
+        }
+      })
+      .map((item: any) => ({
+        prompt: item.value.q,
+        completion: item.value.text,
+        vector: JSON.parse(item.value.rawVector)
+      }));
+
+    jsonRes(res, {
+      data: JSON.stringify(data)
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/getModelData.ts

@@ -0,0 +1,74 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataIdx } from '@/constants/redis';
+import { SearchOptions } from 'redis';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    let {
+      modelId,
+      pageNum = 1,
+      pageSize = 10
+    } = req.query as {
+      modelId: string;
+      pageNum: string;
+      pageSize: string;
+    };
+
+    const { authorization } = req.headers;
+
+    pageNum = +pageNum;
+    pageSize = +pageSize;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+
+    if (!modelId) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const userId = await authToken(authorization);
+
+    await connectToDatabase();
+    const redis = await connectRedis();
+
+    // 从 redis 中获取数据
+    const searchRes = await redis.ft.search(
+      VecModelDataIdx,
+      `@modelId:{${modelId}} @userId:{${userId}}`,
+      {
+        RETURN: ['q', 'text', 'status'],
+        LIMIT: {
+          from: (pageNum - 1) * pageSize,
+          size: pageSize
+        },
+        SORTBY: {
+          BY: 'modelId',
+          DIRECTION: 'DESC'
+        }
+      }
+    );
+
+    jsonRes(res, {
+      data: {
+        pageNum,
+        pageSize,
+        data: searchRes.documents.map((item) => ({
+          id: item.id,
+          ...item.value
+        })),
+        total: searchRes.total
+      }
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/getSplitData.ts

@@ -0,0 +1,35 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, SplitData, Model } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+
+/* 拆分数据成QA */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { modelId } = req.query as { modelId: string };
+    if (!modelId) {
+      throw new Error('参数错误');
+    }
+    await connectToDatabase();
+
+    const { authorization } = req.headers;
+
+    const userId = await authToken(authorization);
+
+    // 找到长度大于0的数据
+    const data = await SplitData.find({
+      userId,
+      modelId,
+      textList: { $exists: true, $not: { $size: 0 } }
+    });
+
+    jsonRes(res, {
+      data
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/pushModelDataInput.ts

@@ -0,0 +1,72 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Model } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { ModelDataSchema } from '@/types/mongoSchema';
+import { generateVector } from '@/service/events/generateVector';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataPrefix, ModelDataStatusEnum } from '@/constants/redis';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    const { modelId, data } = req.body as {
+      modelId: string;
+      data: { text: ModelDataSchema['text']; q: ModelDataSchema['q'] }[];
+    };
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+
+    if (!modelId || !Array.isArray(data)) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const userId = await authToken(authorization);
+
+    await connectToDatabase();
+    const redis = await connectRedis();
+
+    // 验证是否是该用户的 model
+    const model = await Model.findOne({
+      _id: modelId,
+      userId
+    });
+
+    if (!model) {
+      throw new Error('无权操作该模型');
+    }
+
+    const insertRes = await Promise.allSettled(
+      data.map((item) => {
+        return redis.sendCommand([
+          'HMSET',
+          `${VecModelDataPrefix}:${item.q.id}`,
+          'userId',
+          userId,
+          'modelId',
+          modelId,
+          'q',
+          item.q.text,
+          'text',
+          item.text,
+          'status',
+          ModelDataStatusEnum.waiting
+        ]);
+      })
+    );
+
+    generateVector(true);
+
+    jsonRes(res, {
+      data: insertRes.filter((item) => item.status === 'rejected').length
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/pushModelDataJson.ts

@@ -0,0 +1,80 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, Model } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { generateVector } from '@/service/events/generateVector';
+import { vectorToBuffer, formatVector } from '@/utils/tools';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataPrefix, ModelDataStatusEnum } from '@/constants/redis';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    const { modelId, data } = req.body as {
+      modelId: string;
+      data: { prompt: string; completion: string; vector?: number[] }[];
+    };
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+
+    if (!modelId || !Array.isArray(data)) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const userId = await authToken(authorization);
+
+    await connectToDatabase();
+    const redis = await connectRedis();
+
+    // 验证是否是该用户的 model
+    const model = await Model.findOne({
+      _id: modelId,
+      userId
+    });
+
+    if (!model) {
+      throw new Error('无权操作该模型');
+    }
+
+    // 插入 redis
+    const insertRedisRes = await Promise.allSettled(
+      data.map((item) => {
+        const vector = item.vector;
+
+        return redis.sendCommand([
+          'HMSET',
+          `${VecModelDataPrefix}:${nanoid()}`,
+          'userId',
+          userId,
+          'modelId',
+          String(modelId),
+          ...(vector
+            ? ['vector', vectorToBuffer(formatVector(vector)), 'rawVector', JSON.stringify(vector)]
+            : []),
+          'q',
+          item.prompt,
+          'text',
+          item.completion,
+          'status',
+          vector ? ModelDataStatusEnum.ready : ModelDataStatusEnum.waiting
+        ]);
+      })
+    );
+
+    generateVector(true);
+
+    jsonRes(res, {
+      data: insertRedisRes.filter((item) => item.status === 'rejected').length
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/putModelData.ts

@@ -0,0 +1,52 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { authToken } from '@/service/utils/tools';
+import { connectRedis } from '@/service/redis';
+import { ModelDataStatusEnum } from '@/constants/redis';
+import { generateVector } from '@/service/events/generateVector';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    const { dataId, text, q } = req.body as { dataId: string; text: string; q?: string };
+    const { authorization } = req.headers;
+
+    if (!authorization) {
+      throw new Error('无权操作');
+    }
+
+    if (!dataId) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const userId = await authToken(authorization);
+
+    const redis = await connectRedis();
+
+    // 校验是否为该用户的数据
+    const dataItemUserId = await redis.hGet(dataId, 'userId');
+    if (dataItemUserId !== userId) {
+      throw new Error('无权操作');
+    }
+
+    // 更新
+    await redis.sendCommand([
+      'HMSET',
+      dataId,
+      ...(q ? ['q', q, 'status', ModelDataStatusEnum.waiting] : []),
+      'text',
+      text
+    ]);
+
+    if (q) {
+      generateVector();
+    }
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/data/splitData.ts

@@ -0,0 +1,78 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase, SplitData, Model } from '@/service/mongo';
+import { authToken } from '@/service/utils/tools';
+import { generateQA } from '@/service/events/generateQA';
+import { encode } from 'gpt-token-utils';
+
+/* 拆分数据成QA */
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { text, modelId, prompt } = req.body as { text: string; modelId: string; prompt: string };
+    if (!text || !modelId || !prompt) {
+      throw new Error('参数错误');
+    }
+    await connectToDatabase();
+
+    const { authorization } = req.headers;
+
+    const userId = await authToken(authorization);
+
+    // 验证是否是该用户的 model
+    const model = await Model.findOne({
+      _id: modelId,
+      userId
+    });
+
+    if (!model) {
+      throw new Error('无权操作该模型');
+    }
+
+    const replaceText = text.replace(/(\\n|\n)+/g, ' ');
+
+    // 文本拆分成 chunk
+    const chunks = replaceText.match(/[^!?.。]+[!?.。]/g) || [];
+
+    const textList: string[] = [];
+    let splitText = '';
+
+    /* 取 3k ~ 4K tokens 内容 */
+    chunks.forEach((chunk) => {
+      const tokens = encode(splitText + chunk).length;
+      if (tokens >= 4000) {
+        // 超过 4000，不要这块内容
+        textList.push(splitText);
+        splitText = chunk;
+      } else if (tokens >= 3000) {
+        // 超过 3000，取内容
+        textList.push(splitText + chunk);
+        splitText = '';
+      } else {
+        //没超过 3000，继续添加
+        splitText += chunk;
+      }
+    });
+
+    if (splitText) {
+      textList.push(splitText);
+    }
+
+    // 批量插入数据
+    await SplitData.create({
+      userId,
+      modelId,
+      rawText: text,
+      textList,
+      prompt
+    });
+
+    generateQA();
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

src/pages/api/model/del.ts

@@ -1,11 +1,13 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Chat, Model, Training, connectToDatabase } from '@/service/mongo';
-import { authToken, getUserOpenaiKey } from '@/service/utils/tools';
+import { authToken } from '@/service/utils/tools';
+import { getUserApiOpenai } from '@/service/utils/openai';
 import { TrainingStatusEnum } from '@/constants/model';
-import { getOpenAIApi } from '@/service/utils/chat';
 import { TrainingItemType } from '@/types/training';
 import { httpsAgent } from '@/service/utils/tools';
+import { connectRedis } from '@/service/redis';
+import { VecModelDataIdx } from '@/constants/redis';
 
 /* 获取我的模型 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -24,14 +26,33 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const userId = await authToken(authorization);
 
-    await connectToDatabase();
+    // 验证是否是该用户的 model
-
+    const model = await Model.findOne({
-    // 删除模型
-    await Model.deleteOne({
       _id: modelId,
       userId
     });
 
+    if (!model) {
+      throw new Error('无权操作该模型');
+    }
+
+    await connectToDatabase();
+    const redis = await connectRedis();
+
+    // 获取 redis 中模型关联的所有数据
+    const searchRes = await redis.ft.search(
+      VecModelDataIdx,
+      `@modelId:{${modelId}} @userId:{${userId}}`,
+      {
+        LIMIT: {
+          from: 0,
+          size: 10000
+        }
+      }
+    );
+    // 删除 redis 内容
+    await Promise.all(searchRes.documents.map((item) => redis.del(item.id)));
+
     // 删除对应的聊天
     await Chat.deleteMany({
       modelId
@@ -45,7 +66,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     // 如果正在训练，需要删除openai上的相关信息
     if (training) {
-      const openai = getOpenAIApi(await getUserOpenaiKey(userId));
+      const { openai } = await getUserApiOpenai(userId);
       // 获取训练记录
       const tuneRecord = await openai.retrieveFineTune(training.tuneId, { httpsAgent });
 
@@ -60,6 +81,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       modelId
     });
 
+    // 删除模型
+    await Model.deleteOne({
+      _id: modelId,
+      userId
+    });
+
     jsonRes(res);
   } catch (err) {
     jsonRes(res, {

src/pages/api/model/train/getTrainings.ts

@@ -1,15 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, Model, Training } from '@/service/mongo';
+import { connectToDatabase, Training } from '@/service/mongo';
-import { getOpenAIApi } from '@/service/utils/chat';
+import { authToken } from '@/service/utils/tools';
-import formidable from 'formidable';
-import { authToken, getUserOpenaiKey } from '@/service/utils/tools';
-import { join } from 'path';
-import fs from 'fs';
-import type { ModelSchema } from '@/types/mongoSchema';
-import type { OpenAIApi } from 'openai';
-import { ModelStatusEnum, TrainingStatusEnum } from '@/constants/model';
-import { httpsAgent } from '@/service/utils/tools';
 
 // 关闭next默认的bodyParser处理方式
 export const config = {
@@ -18,7 +10,7 @@ export const config = {
   }
 };
 
-/* 上传文件，开始微调 */
+/* 获取模型训练记录 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     const { authorization } = req.headers;
@@ -30,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     if (!modelId) {
       throw new Error('参数错误');
     }
-    const userId = await authToken(authorization);
+    await authToken(authorization);
 
     await connectToDatabase();
 

src/pages/api/model/train/putTrainStatus.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Model, Training } from '@/service/mongo';
-import { getOpenAIApi } from '@/service/utils/chat';
+import { authToken } from '@/service/utils/tools';
-import { authToken, getUserOpenaiKey } from '@/service/utils/tools';
+import { getUserApiOpenai } from '@/service/utils/openai';
 import type { ModelSchema } from '@/types/mongoSchema';
 import { TrainingItemType } from '@/types/training';
 import { ModelStatusEnum, TrainingStatusEnum } from '@/constants/model';
@@ -43,16 +43,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     }
 
     // 用户的 openai 实例
-    const openai = getOpenAIApi(await getUserOpenaiKey(userId));
+    const { openai } = await getUserApiOpenai(userId);
 
     // 获取 openai 的训练情况
     const { data } = await openai.retrieveFineTune(training.tuneId, { httpsAgent });
-
+    // console.log(data);
     if (data.status === OpenAiTuneStatusEnum.succeeded) {
       // 删除训练文件
       openai.deleteFile(data.training_files[0].id, { httpsAgent });
 
-      // 更新模型
+      // 更新模型状态和模型内容
       await Model.findByIdAndUpdate(modelId, {
         status: ModelStatusEnum.running,
         updateTime: new Date(),
@@ -72,6 +72,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       });
     }
 
+    /* 取消微调 */
     if (data.status === OpenAiTuneStatusEnum.cancelled) {
       // 删除训练文件
       openai.deleteFile(data.training_files[0].id, { httpsAgent });
@@ -87,11 +88,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       });
 
       return jsonRes(res, {
-        data: '模型微调取消'
+        data: '模型微调已取消'
       });
     }
 
-    throw new Error('模型还在训练中');
+    jsonRes(res, {
+      data: '模型还在训练中'
+    });
   } catch (err: any) {
     jsonRes(res, {
       code: 500,

src/pages/api/model/train/train.ts

@@ -2,9 +2,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Model, Training } from '@/service/mongo';
-import { getOpenAIApi } from '@/service/utils/chat';
 import formidable from 'formidable';
-import { authToken, getUserOpenaiKey } from '@/service/utils/tools';
+import { authToken } from '@/service/utils/tools';
+import { getUserApiOpenai } from '@/service/utils/openai';
 import { join } from 'path';
 import fs from 'fs';
 import type { ModelSchema } from '@/types/mongoSchema';
@@ -30,6 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       throw new Error('无权操作');
     }
     const { modelId } = req.query;
+
     if (!modelId) {
       throw new Error('参数错误');
     }
@@ -48,7 +49,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const trainingType = model.service.trainId; // 目前都默认是 openai text-davinci-03
 
     // 获取用户的 API Key 实例化后的对象
-    openai = getOpenAIApi(await getUserOpenaiKey(userId));
+    const user = await getUserApiOpenai(userId);
+    openai = user.openai;
 
     // 接收文件并保存
     const form = formidable({
@@ -67,7 +69,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     });
     const file = files.file;
 
-    // 上传文件
+    // 上传文件到 openai
     // @ts-ignore
     const uploadRes = await openai.createFile(
       // @ts-ignore
@@ -82,7 +84,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       {
         training_file: uploadFileId,
         model: trainingType,
-        suffix: model.name
+        suffix: model.name,
+        n_epochs: 4
       },
       { httpsAgent }
     );

src/pages/api/model/update.ts

@@ -37,7 +37,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         systemPrompt,
         intro,
         temperature,
-        service,
+        // service,
         security
       }
     );

src/pages/api/timer/clearAuthCode.ts

@@ -5,8 +5,8 @@ import { AuthCode } from '@/service/models/authCode';
 import { connectToDatabase } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  if (req.headers.auth !== 'archer') {
+  if (process.env.NODE_ENV !== 'development') {
-    throw new Error('凭证错误');
+    throw new Error('不是开发环境');
   }
   try {
     await connectToDatabase();

src/pages/api/timer/clearChatWindow.ts

@@ -4,8 +4,8 @@ import { connectToDatabase, Chat } from '@/service/mongo';
 
 /* 定时删除那些不活跃的内容 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  if (req.headers.auth !== 'archer') {
+  if (process.env.NODE_ENV !== 'development') {
-    throw new Error('凭证错误');
+    throw new Error('不是开发环境');
   }
   try {
     await connectToDatabase();

src/pages/api/timer/initBill.ts

@@ -7,8 +7,8 @@ import type { BillSchema } from '@/types/mongoSchema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    if (req.headers.auth !== 'archer') {
+    if (process.env.NODE_ENV !== 'development') {
-      throw new Error('凭证错误');
+      throw new Error('不是开发环境');
     }
     await connectToDatabase();
 

src/pages/api/timer/initDataItemTime.ts

@@ -5,22 +5,23 @@ import { connectToDatabase, DataItem, Data } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    if (req.headers.auth !== 'archer') {
+    if (process.env.NODE_ENV !== 'development') {
-      throw new Error('凭证错误');
+      throw new Error('不是开发环境');
     }
     await connectToDatabase();
 
     // await DataItem.updateMany(
     //   {},
     //   {
-    //     times: 2
+    //     type: 'QA'
+    //     // times: 2
     //   }
     // );
 
     await Data.updateMany(
       {},
       {
-        isDeleted: false
+        type: 'QA'
       }
     );
 

src/pages/api/timer/updateTraining.ts

@@ -4,16 +4,15 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase, Training, Model } from '@/service/mongo';
 import type { TrainingItemType } from '@/types/training';
 import { TrainingStatusEnum, ModelStatusEnum } from '@/constants/model';
-import { getOpenAIApi } from '@/service/utils/chat';
+import { getUserApiOpenai } from '@/service/utils/openai';
-import { getUserOpenaiKey } from '@/service/utils/tools';
 import { OpenAiTuneStatusEnum } from '@/service/constants/training';
 import { sendTrainSucceed } from '@/service/utils/sendEmail';
 import { httpsAgent } from '@/service/utils/tools';
 import { ModelPopulate } from '@/types/mongoSchema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  if (req.headers.auth !== 'archer') {
+  if (process.env.NODE_ENV !== 'development') {
-    throw new Error('凭证错误');
+    throw new Error('不是开发环境');
   }
   try {
     await connectToDatabase();
@@ -23,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       status: TrainingStatusEnum.pending
     });
 
-    const openai = getOpenAIApi(await getUserOpenaiKey('63f9a14228d2a688d8dc9e1b'));
+    const { openai } = await getUserApiOpenai('63f9a14228d2a688d8dc9e1b');
 
     const response = await Promise.all(
       trainingRecords.map(async (item) => {

src/pages/chat/components/Empty.tsx

@@ -25,14 +25,13 @@ const Empty = ({ intro }: { intro: string }) => {
           <Box whiteSpace={'pre-line'}>{intro}</Box>
         </Card>
       )}
-      <Card p={4} mb={10}>
-        <Header>常见问题</Header>
-        <Markdown source={chatProblem} />
-      </Card>
       {/* version intro */}
-      <Card p={4}>
+      <Card p={4} mb={10}>
         <Markdown source={versionIntro} />
       </Card>
+      <Card p={4}>
+        <Markdown source={chatProblem} />
+      </Card>
     </Box>
   );
 };

src/pages/chat/components/SlideBar.tsx

@@ -231,12 +231,12 @@ const SlideBar = ({
         </>
       </RenderButton>
 
-      <RenderButton onClick={onOpenShare}>
+      {/* <RenderButton onClick={onOpenShare}>
         <>
           <MyIcon name="share" fill={'white'} w={'16px'} h={'16px'} mr={4} />
           分享
         </>
-      </RenderButton>
+      </RenderButton> */}
       <RenderButton onClick={() => router.push('/number/setting')}>
         <>
           <MyIcon name="pay" fill={'white'} w={'16px'} h={'16px'} mr={4} />

src/pages/chat/index.tsx

@@ -5,7 +5,7 @@ import {
   getInitChatSiteInfo,
   getChatSiteId,
   postGPT3SendPrompt,
-  delLastMessage,
+  delChatRecordByIndex,
   postSaveChat
 } from '@/api/chat';
 import type { InitChatResponse } from '@/api/response/chat';
@@ -14,26 +14,30 @@ import {
   Textarea,
   Box,
   Flex,
-  Button,
   useDisclosure,
   Drawer,
   DrawerOverlay,
   DrawerContent,
-  useColorModeValue
+  useColorModeValue,
+  Menu,
+  MenuButton,
+  MenuList,
+  MenuItem
 } from '@chakra-ui/react';
 import { useToast } from '@/hooks/useToast';
-import Icon from '@/components/Iconfont';
 import { useScreen } from '@/hooks/useScreen';
 import { useQuery } from '@tanstack/react-query';
 import { ChatModelNameEnum } from '@/constants/model';
 import dynamic from 'next/dynamic';
 import { useGlobalStore } from '@/store/global';
 import { useChatStore } from '@/store/chat';
+import { useCopyData } from '@/utils/tools';
 import { streamFetch } from '@/api/fetch';
 import SlideBar from './components/SlideBar';
 import Empty from './components/Empty';
-import { getToken } from '@/utils/user';
+import Icon from '@/components/Icon';
-import MyIcon from '@/components/Icon';
+import { encode } from 'gpt-token-utils';
+import { modelList } from '@/constants/model';
 
 const Markdown = dynamic(() => import('@/components/Markdown'));
 
@@ -46,58 +50,32 @@ interface ChatType extends InitChatResponse {
 const Chat = ({ chatId }: { chatId: string }) => {
   const { toast } = useToast();
   const router = useRouter();
-  const { isPc, media } = useScreen();
+  const ChatBox = useRef<HTMLDivElement>(null);
-  const { setLoading } = useGlobalStore();
+  const TextareaDom = useRef<HTMLTextAreaElement>(null);
+  // 中断请求
+  const controller = useRef(new AbortController());
   const [chatData, setChatData] = useState<ChatType>({
     chatId: '',
     modelId: '',
     name: '',
     avatar: '',
     intro: '',
-    secret: {},
     chatModel: '',
-    history: [],
+    modelName: '',
-    isExpiredTime: false
+    history: []
   }); // 聊天框整体数据
-
-  const ChatBox = useRef<HTMLDivElement>(null);
-  const TextareaDom = useRef<HTMLTextAreaElement>(null);
-
   const [inputVal, setInputVal] = useState(''); // 输入的内容
-  const { isOpen: isOpenSlider, onClose: onCloseSlider, onOpen: onOpenSlider } = useDisclosure();
 
   const isChatting = useMemo(
     () => chatData.history[chatData.history.length - 1]?.status === 'loading',
     [chatData.history]
   );
-  const chatWindowError = useMemo(() => {
+  const { copyData } = useCopyData();
-    if (chatData.history[chatData.history.length - 1]?.obj === 'Human') {
+  const { isPc, media } = useScreen();
-      return {
+  const { setLoading } = useGlobalStore();
-        text: '内容出现异常',
-        canDelete: true
-      };
-    }
-    if (chatData.isExpiredTime) {
-      return {
-        text: '聊天框已过期',
-        canDelete: false
-      };
-    }
-
-    return '';
-  }, [chatData]);
 
+  const { isOpen: isOpenSlider, onClose: onCloseSlider, onOpen: onOpenSlider } = useDisclosure();
   const { pushChatHistory } = useChatStore();
-  // 中断请求
-  const controller = useRef(new AbortController());
-  useEffect(() => {
-    controller.current = new AbortController();
-    return () => {
-      console.log('close========');
-      // eslint-disable-next-line react-hooks/exhaustive-deps
-      controller.current?.abort();
-    };
-  }, [chatId]);
 
   // 滚动到底部
   const scrollToBottom = useCallback(() => {
@@ -110,42 +88,6 @@ const Chat = ({ chatId }: { chatId: string }) => {
     }, 100);
   }, []);
 
-  // 初始化聊天框
-  useQuery(
-    ['init', chatId],
-    () => {
-      setLoading(true);
-      return getInitChatSiteInfo(chatId);
-    },
-    {
-      onSuccess(res) {
-        setChatData({
-          ...res,
-          history: res.history.map((item) => ({
-            ...item,
-            status: 'finish'
-          }))
-        });
-        if (res.history.length > 0) {
-          setTimeout(() => {
-            scrollToBottom();
-          }, 500);
-        }
-      },
-      onError(e: any) {
-        toast({
-          title: e?.message || '初始化异常,请检查地址',
-          status: 'error',
-          isClosable: true,
-          duration: 5000
-        });
-      },
-      onSettled() {
-        setLoading(false);
-      }
-    }
-  );
-
   // 重置输入内容
   const resetInputVal = useCallback((val: string) => {
     setInputVal(val);
@@ -172,39 +114,17 @@ const Chat = ({ chatId }: { chatId: string }) => {
     onCloseSlider();
   }, [chatData, onCloseSlider, router, toast]);
 
-  // gpt3 方法
-  const gpt3ChatPrompt = useCallback(
-    async (newChatList: ChatSiteItemType[]) => {
-      // 请求内容
-      const response = await postGPT3SendPrompt({
-        prompt: newChatList,
-        chatId: chatId as string
-      });
-
-      // 更新 AI 的内容
-      setChatData((state) => ({
-        ...state,
-        history: state.history.map((item, index) => {
-          if (index !== state.history.length - 1) return item;
-          return {
-            ...item,
-            status: 'finish',
-            value: response
-          };
-        })
-      }));
-    },
-    [chatId]
-  );
-
   // gpt 对话
   const gptChatPrompt = useCallback(
     async (prompts: ChatSiteItemType) => {
       const urlMap: Record<string, string> = {
         [ChatModelNameEnum.GPT35]: '/api/chat/chatGpt',
+        [ChatModelNameEnum.VECTOR_GPT]: '/api/chat/vectorGpt',
+        // [ChatModelNameEnum.VECTOR_GPT]: '/api/chat/lafGpt',
         [ChatModelNameEnum.GPT3]: '/api/chat/gpt3'
       };
-      if (!urlMap[chatData.chatModel]) return Promise.reject('找不到模型');
+
+      if (!urlMap[chatData.modelName]) return Promise.reject('找不到模型');
 
       const prompt = {
         obj: prompts.obj,
@@ -212,7 +132,7 @@ const Chat = ({ chatId }: { chatId: string }) => {
       };
       // 流请求，获取数据
       const res = await streamFetch({
-        url: urlMap[chatData.chatModel],
+        url: urlMap[chatData.modelName],
         data: {
           prompt,
           chatId
@@ -265,21 +185,41 @@ const Chat = ({ chatId }: { chatId: string }) => {
         })
       }));
     },
-    [chatData.chatModel, chatId, toast]
+    [chatData.modelName, chatId, toast]
   );
 
   /**
    * 发送一个内容
    */
   const sendPrompt = useCallback(async () => {
+    if (isChatting) {
+      toast({
+        title: '正在聊天中...请等待结束',
+        status: 'warning'
+      });
+      return;
+    }
     const storeInput = inputVal;
     // 去除空行
-    const val = inputVal
+    const val = inputVal.trim().replace(/\n\s*/g, '\n');
-      .trim()
+
-      .split('\n')
+    if (!chatData?.modelId || !val) {
-      .filter((val) => val)
+      toast({
-      .join('\n');
+        title: '内容为空',
-    if (!chatData?.modelId || !val || !ChatBox.current || isChatting) {
+        status: 'warning'
+      });
+      return;
+    }
+
+    // 长度校验
+    const tokens = encode(val).length;
+    const model = modelList.find((item) => item.model === chatData.modelName);
+
+    if (model && tokens >= model.maxToken) {
+      toast({
+        title: '单次输入超出 4000 tokens',
+        status: 'warning'
+      });
       return;
     }
 
@@ -335,31 +275,89 @@ const Chat = ({ chatId }: { chatId: string }) => {
     }
   }, [
     inputVal,
-    chatData?.modelId,
+    chatData,
-    chatData.history,
     isChatting,
     resetInputVal,
     scrollToBottom,
+    toast,
     gptChatPrompt,
     pushChatHistory,
-    chatId,
+    chatId
-    toast
   ]);
 
-  // 重新编辑
+  // 删除一句话
-  const reEdit = useCallback(async () => {
+  const delChatRecord = useCallback(
-    if (chatData.history[chatData.history.length - 1]?.obj !== 'Human') return;
+    async (index: number) => {
+      setLoading(true);
+      try {
         // 删除数据库最后一句
-    await delLastMessage(chatId);
+        await delChatRecordByIndex(chatId, index);
-    const val = chatData.history[chatData.history.length - 1].value;
-
-    resetInputVal(val);
 
         setChatData((state) => ({
           ...state,
-      history: state.history.slice(0, -1)
+          history: state.history.filter((_, i) => i !== index)
         }));
-  }, [chatData.history, chatId, resetInputVal]);
+      } catch (err) {
+        console.log(err);
+      }
+      setLoading(false);
+    },
+    [chatId, setLoading]
+  );
+
+  // 复制内容
+  const onclickCopy = useCallback(
+    (chatId: string) => {
+      const dom = document.getElementById(chatId);
+      const innerText = dom?.innerText;
+      innerText && copyData(innerText);
+    },
+    [copyData]
+  );
+
+  useEffect(() => {
+    controller.current = new AbortController();
+    return () => {
+      // eslint-disable-next-line react-hooks/exhaustive-deps
+      controller.current?.abort();
+    };
+  }, [chatId]);
+
+  // 初始化聊天框
+  useQuery(
+    ['init', chatId],
+    () => {
+      setLoading(true);
+      return getInitChatSiteInfo(chatId);
+    },
+    {
+      onSuccess(res) {
+        setChatData({
+          ...res,
+          history: res.history.map((item) => ({
+            ...item,
+            status: 'finish'
+          }))
+        });
+        if (res.history.length > 0) {
+          setTimeout(() => {
+            scrollToBottom();
+          }, 500);
+        }
+      },
+      onError(e: any) {
+        toast({
+          title: e?.message || '初始化异常,请检查地址',
+          status: 'error',
+          isClosable: true,
+          duration: 5000
+        });
+      },
+      onSettled() {
+        setLoading(false);
+      }
+    }
+  );
 
   return (
     <Flex
@@ -389,7 +387,7 @@ const Chat = ({ chatId }: { chatId: string }) => {
             px={7}
           >
             <Box onClick={onOpenSlider}>
-              <MyIcon
+              <Icon
                 name={'menu'}
                 w={'20px'}
                 h={'20px'}
@@ -419,7 +417,7 @@ const Chat = ({ chatId }: { chatId: string }) => {
         flexDirection={'column'}
       >
         {/* 聊天内容 */}
-        <Box ref={ChatBox} flex={'1 0 0'} h={0} w={'100%'} overflowY={'auto'}>
+        <Box ref={ChatBox} pb={[4, 0]} flex={'1 0 0'} h={0} w={'100%'} overflowY={'auto'}>
           {chatData.history.map((item, index) => (
             <Box
               key={index}
@@ -432,15 +430,21 @@ const Chat = ({ chatId }: { chatId: string }) => {
               borderBottom={'1px solid rgba(0,0,0,0.1)'}
             >
               <Flex maxW={'750px'} m={'auto'} alignItems={'flex-start'}>
-                <Box mr={media(4, 1)}>
+                <Menu>
+                  <MenuButton as={Box} mr={media(4, 1)} cursor={'pointer'}>
                     <Image
                       src={item.obj === 'Human' ? '/icon/human.png' : '/icon/logo.png'}
                       alt="/icon/logo.png"
                       width={media(30, 20)}
                       height={media(30, 20)}
                     />
-                </Box>
+                  </MenuButton>
-                <Box flex={'1 0 0'} w={0} overflow={'hidden'}>
+                  <MenuList fontSize={'sm'}>
+                    <MenuItem onClick={() => onclickCopy(`chat${index}`)}>复制</MenuItem>
+                    <MenuItem onClick={() => delChatRecord(index)}>删除该行</MenuItem>
+                  </MenuList>
+                </Menu>
+                <Box flex={'1 0 0'} w={0} overflow={'hidden'} id={`chat${index}`}>
                   {item.obj === 'AI' ? (
                     <Markdown
                       source={item.value}
@@ -457,22 +461,8 @@ const Chat = ({ chatId }: { chatId: string }) => {
         </Box>
         {/* 发送区 */}
         <Box m={media('20px auto', '0 auto')} w={'100%'} maxW={media('min(750px, 100%)', 'auto')}>
-          {!!chatWindowError ? (
-            <Box textAlign={'center'}>
-              <Box color={'red'}>{chatWindowError.text}</Box>
-              <Flex py={5} justifyContent={'center'}>
-                {getToken() && <Button onClick={resetChat}>重开对话</Button>}
-
-                {chatWindowError.canDelete && (
-                  <Button ml={20} colorScheme={'green'} onClick={reEdit}>
-                    重新编辑最后一句
-                  </Button>
-                )}
-              </Flex>
-            </Box>
-          ) : (
           <Box
-              py={5}
+            py={'18px'}
             position={'relative'}
             boxShadow={`0 0 15px rgba(0,0,0,0.1)`}
             border={media('1px solid', '0')}
@@ -483,9 +473,8 @@ const Chat = ({ chatId }: { chatId: string }) => {
             {/* 输入框 */}
             <Textarea
               ref={TextareaDom}
-                w={'100%'}
-                pr={'45px'}
               py={0}
+              pr={['45px', '55px']}
               border={'none'}
               _focusVisible={{
                 border: 'none'
@@ -497,8 +486,10 @@ const Chat = ({ chatId }: { chatId: string }) => {
               height={'22px'}
               lineHeight={'22px'}
               maxHeight={'150px'}
-                maxLength={chatData?.secret.contentMaxLen || -1}
+              maxLength={-1}
               overflowY={'auto'}
+              whiteSpace={'pre-wrap'}
+              wordBreak={'break-all'}
               color={useColorModeValue('blackAlpha.700', 'white')}
               onChange={(e) => {
                 const textarea = e.target;
@@ -518,29 +509,35 @@ const Chat = ({ chatId }: { chatId: string }) => {
               }}
             />
             {/* 发送和等待按键 */}
-              <Box position={'absolute'} bottom={5} right={media('20px', '10px')}>
+            <Flex
+              alignItems={'center'}
+              justifyContent={'center'}
+              h={'30px'}
+              w={'30px'}
+              position={'absolute'}
+              right={['12px', '20px']}
+              bottom={'15px'}
+              onClick={sendPrompt}
+            >
               {isChatting ? (
                 <Image
                   style={{ transform: 'translateY(4px)' }}
                   src={'/icon/chatting.svg'}
-                    width={30}
+                  fill
-                    height={30}
                   alt={''}
                 />
               ) : (
-                  <Box cursor={'pointer'} onClick={sendPrompt}>
                 <Icon
-                      name={'icon-fasong'}
+                  name={'chatSend'}
-                      width={20}
+                  width={['18px', '20px']}
-                      height={20}
+                  height={['18px', '20px']}
-                      color={useColorModeValue('#718096', 'white')}
+                  cursor={'pointer'}
+                  fill={useColorModeValue('#718096', 'white')}
                 ></Icon>
-                  </Box>
               )}
+            </Flex>
           </Box>
         </Box>
-          )}
-        </Box>
       </Flex>
     </Flex>
   );

src/pages/data/components/CreateDataModal.tsx

@@ -8,10 +8,21 @@ import {
   ModalBody,
   ModalCloseButton,
   Button,
-  Input
+  Input,
+  Select,
+  FormControl,
+  FormErrorMessage
 } from '@chakra-ui/react';
 import { postData } from '@/api/data';
 import { useMutation } from '@tanstack/react-query';
+import { useForm, SubmitHandler } from 'react-hook-form';
+import { DataType } from '@/types/data';
+import { DataTypeTextMap } from '@/constants/data';
+
+export interface CreateDataProps {
+  name: string;
+  type: DataType;
+}
 
 const CreateDataModal = ({
   onClose,
@@ -21,9 +32,20 @@ const CreateDataModal = ({
   onSuccess: () => void;
 }) => {
   const [inputVal, setInputVal] = useState('');
+  const {
+    getValues,
+    register,
+    handleSubmit,
+    formState: { errors }
+  } = useForm<CreateDataProps>({
+    defaultValues: {
+      name: '',
+      type: 'abstract'
+    }
+  });
 
   const { isLoading, mutate } = useMutation({
-    mutationFn: (name: string) => postData(name),
+    mutationFn: (e: CreateDataProps) => postData(e),
     onSuccess() {
       onSuccess();
       onClose();
@@ -37,23 +59,33 @@ const CreateDataModal = ({
         <ModalHeader>创建数据集</ModalHeader>
         <ModalCloseButton />
 
-        <ModalBody display={'flex'}>
+        <ModalBody>
+          <FormControl mb={8} isInvalid={!!errors.name}>
             <Input
-            value={inputVal}
+              placeholder="数据集名称"
-            onChange={(e) => setInputVal(e.target.value)}
+              {...register('name', {
-            placeholder={'数据集名称'}
+                required: '数据集名称不能为空'
-          ></Input>
+              })}
+            />
+            <FormErrorMessage position={'absolute'} fontSize="xs">
+              {!!errors.name && errors.name.message}
+            </FormErrorMessage>
+          </FormControl>
+          <FormControl>
+            <Select placeholder="数据集类型" {...register('type', {})}>
+              {Object.entries(DataTypeTextMap).map(([key, value]) => (
+                <option key={key} value={key}>
+                  {value}
+                </option>
+              ))}
+            </Select>
+          </FormControl>
         </ModalBody>
         <ModalFooter>
           <Button colorScheme={'gray'} onClick={onClose}>
             取消
           </Button>
-          <Button
+          <Button ml={3} isLoading={isLoading} onClick={handleSubmit(mutate as any)}>
-            ml={3}
-            isDisabled={inputVal === ''}
-            isLoading={isLoading}
-            onClick={() => mutate(inputVal)}
-          >
             确认
           </Button>
         </ModalFooter>

src/pages/data/components/ImportDataModal.tsx

@@ -22,6 +22,7 @@ import { useToast } from '@/hooks/useToast';
 import { useLoading } from '@/hooks/useLoading';
 import { formatPrice } from '@/utils/user';
 import { modelList, ChatModelNameEnum } from '@/constants/model';
+import { encode } from 'gpt-token-utils';
 
 const fileExtension = '.txt,.doc,.docx,.pdf,.md';
 
@@ -106,6 +107,7 @@ const ImportDataModal = ({
           .join('\n')
           .replace(/\n+/g, '\n');
         setFileText(fileTexts);
+        console.log(encode(fileTexts));
       } catch (error: any) {
         console.log(error);
         toast({
@@ -161,7 +163,9 @@ const ImportDataModal = ({
                   placeholder={'请粘贴或输入需要处理的文本'}
                   onChange={(e) => setTextInput(e.target.value)}
                 />
-                <Box mt={2}>一共 {textInput.length} 个字</Box>
+                <Box mt={2}>
+                  一共 {textInput.length} 个字，{encode(textInput).length} 个tokens
+                </Box>
               </>
             )}
             {activeTab === 'doc' && (
@@ -174,12 +178,15 @@ const ImportDataModal = ({
                 border={'1px solid '}
                 borderColor={'blackAlpha.200'}
                 borderRadius={'md'}
+                fontSize={'sm'}
               >
                 <Button onClick={onOpen}>选择文件</Button>
                 <Box mt={2}>支持 {fileExtension} 文件</Box>
                 {fileText && (
                   <>
-                    <Box mt={2}>一共 {fileText.length} 个字</Box>
+                    <Box mt={2}>
+                      一共 {fileText.length} 个字，{encode(fileText).length} 个tokens
+                    </Box>
                     <Box
                       maxH={'300px'}
                       w={'100%'}

src/pages/data/detail.tsx

@@ -1,5 +1,5 @@
 import React from 'react';
-import { Box, Card, Table, Thead, Tbody, Tr, Th, Td, TableContainer } from '@chakra-ui/react';
+import { Box, Card } from '@chakra-ui/react';
 import ScrollData from '@/components/ScrollData';
 import { getDataItems } from '@/api/data';
 import { usePaging } from '@/hooks/usePaging';
@@ -22,7 +22,7 @@ const DataDetail = ({ dataName, dataId }: { dataName: string; dataId: string })
   return (
     <Card py={4} h={'100%'} display={'flex'} flexDirection={'column'}>
       <Box px={6} fontSize={'xl'} fontWeight={'bold'}>
-        {dataName} 拆分结果
+        {dataName} 结果
       </Box>
       <ScrollData
         flex={'1 0 0'}
@@ -33,13 +33,19 @@ const DataDetail = ({ dataName, dataId }: { dataName: string; dataId: string })
         requesting={requesting}
         nextPage={nextPage}
         fontSize={'xs'}
+        whiteSpace={'pre-wrap'}
       >
         {dataItems.map((item) => (
           <Box key={item._id}>
             {item.result.map((result, i) => (
               <Box key={i} mb={3}>
+                {item.type === 'QA' && (
+                  <>
                     <Box fontWeight={'bold'}>Q: {result.q}</Box>
                     <Box>A: {result.a}</Box>
+                  </>
+                )}
+                {item.type === 'abstract' && <Box fontSize={'sm'}>{result.abstract}</Box>}
               </Box>
             ))}
           </Box>

src/pages/data/list.tsx

@@ -19,34 +19,25 @@ import {
   MenuItem
 } from '@chakra-ui/react';
 import { getDataList, updateDataName, delData, getDataItems } from '@/api/data';
-import { usePaging } from '@/hooks/usePaging';
 import type { DataListItem } from '@/types/data';
-import ScrollData from '@/components/ScrollData';
 import dayjs from 'dayjs';
 import dynamic from 'next/dynamic';
 import { useRouter } from 'next/router';
 import { useConfirm } from '@/hooks/useConfirm';
 import { useRequest } from '@/hooks/useRequest';
 import { DataItemSchema } from '@/types/mongoSchema';
+import { DataTypeTextMap } from '@/constants/data';
+import { customAlphabet } from 'nanoid';
+import { useQuery } from '@tanstack/react-query';
+const nanoid = customAlphabet('.,', 1);
 
 const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
 const ImportDataModal = dynamic(() => import('./components/ImportDataModal'));
 
-export type ExportDataType = 'jsonl';
+export type ExportDataType = 'jsonl' | 'txt';
 
 const DataList = () => {
   const router = useRouter();
-  const {
-    nextPage,
-    isLoadAll,
-    requesting,
-    data: dataList,
-    getData,
-    initRequesting
-  } = usePaging<DataListItem>({
-    api: getDataList,
-    pageSize: 20
-  });
   const [ImportDataId, setImportDataId] = useState<string>();
   const { openConfirm, ConfirmChild } = useConfirm({
     content: '删除数据集，将删除里面的所有内容，请确认！'
@@ -58,12 +49,16 @@ const DataList = () => {
     onClose: onCloseCreateDataModal
   } = useDisclosure();
 
+  const { data: dataList = [], refetch } = useQuery(['getDataList'], getDataList, {
+    refetchInterval: 10000
+  });
+
   const { mutate: handleDelData, isLoading: isDeleting } = useRequest({
     mutationFn: (dataId: string) => delData(dataId),
     successToast: '删除数据集成功',
     errorToast: '删除数据集异常',
     onSuccess() {
-      getData(1, true);
+      refetch();
     }
   });
 
@@ -82,21 +77,26 @@ const DataList = () => {
       let text = '';
       // 生成 jsonl
       data.forEach((item) => {
+        if (res.type === 'jsonl' && item.q && item.a) {
           const result = JSON.stringify({
-          prompt: `${item.q.toLocaleLowerCase()}</s>`,
+            prompt: `${item.q.toLocaleLowerCase()}${nanoid()}</s>`,
-          completion: ` ${item.a}</s>`
+            completion: ` ${item.a}###`
           });
           text += `${result}\n`;
+        } else if (res.type === 'txt' && item.abstract) {
+          text += `${item.abstract}\n`;
+        }
       });
       // 去掉最后一个 \n
       text = text.substring(0, text.length - 1);
+
       // 导出为文件
       const blob = new Blob([text], { type: 'application/json;charset=utf-8' });
 
       // 创建下载链接
       const downloadLink = document.createElement('a');
       downloadLink.href = window.URL.createObjectURL(blob);
-      downloadLink.download = 'file.jsonl';
+      downloadLink.download = `data.${res.type}`;
 
       // 添加链接到页面并触发下载
       document.body.appendChild(downloadLink);
@@ -114,7 +114,7 @@ const DataList = () => {
               训练数据管理
             </Box>
             <Box fontSize={'xs'} color={'blackAlpha.600'}>
-              允许你将任意文本数据拆分成 QA 的形式。你可以使用这些 QA 去微调你的对话模型。
+              允许你将任意文本数据拆分成 QA 形式，或者进行文本摘要总结。
             </Box>
           </Box>
           <Button variant={'outline'} onClick={onOpenCreateDataModal}>
@@ -123,19 +123,22 @@ const DataList = () => {
         </Flex>
       </Card>
       {/* 数据表 */}
-      <Card mt={3} flex={'1 0 0'} h={['auto', '0']} px={6} py={4}>
+      <TableContainer
-        <ScrollData
+        mt={3}
-          h={'100%'}
+        flex={'1 0 0'}
-          nextPage={nextPage}
+        h={['auto', '0']}
-          isLoadAll={isLoadAll}
+        overflowY={'auto'}
-          requesting={requesting}
+        px={6}
-          initRequesting={initRequesting}
+        py={4}
+        backgroundColor={'white'}
+        borderRadius={'md'}
+        boxShadow={'base'}
       >
-          <TableContainer>
         <Table>
           <Thead>
             <Tr>
               <Th>集合名</Th>
+              <Th>类型</Th>
               <Th>创建时间</Th>
               <Th>训练中 / 总数据</Th>
               <Th></Th>
@@ -156,6 +159,7 @@ const DataList = () => {
                     }}
                   />
                 </Td>
+                <Td>{DataTypeTextMap[item.type || 'QA']}</Td>
                 <Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
                 <Td>
                   {item.trainingData} / {item.totalData}
@@ -180,16 +184,23 @@ const DataList = () => {
                   >
                     导入
                   </Button>
-                      <Menu>
+                  {/* <Menu>
-                        <MenuButton as={Button} mr={2} size={'sm'}>
+                    <MenuButton as={Button} mr={2} size={'sm'} isLoading={isExporting}>
                       导出
                     </MenuButton>
                     <MenuList>
+                      {item.type === 'QA' && (
                         <MenuItem onClick={() => handleExportData({ data: item, type: 'jsonl' })}>
                           jsonl
                         </MenuItem>
+                      )}
+                      {item.type === 'abstract' && (
+                        <MenuItem onClick={() => handleExportData({ data: item, type: 'txt' })}>
+                          txt
+                        </MenuItem>
+                      )}
                     </MenuList>
-                      </Menu>
+                  </Menu> */}
 
                   <Button
                     size={'sm'}
@@ -205,18 +216,16 @@ const DataList = () => {
           </Tbody>
         </Table>
       </TableContainer>
-        </ScrollData>
-      </Card>
 
       {ImportDataId && (
         <ImportDataModal
           dataId={ImportDataId}
           onClose={() => setImportDataId(undefined)}
-          onSuccess={() => getData(1, true)}
+          onSuccess={refetch}
         />
       )}
       {isOpenCreateDataModal && (
-        <CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
+        <CreateDataModal onClose={onCloseCreateDataModal} onSuccess={refetch} />
       )}
       <ConfirmChild />
     </Box>

src/pages/login/index.tsx

@@ -71,7 +71,6 @@ const Login = () => {
           order={1}
           flex={`0 0 ${isPc ? '400px' : '100%'}`}
           height={'100%'}
-          maxH={'450px'}
           border="1px"
           borderColor="gray.200"
           py={5}

src/pages/model/detail/components/InputDataModal.tsx

@@ -0,0 +1,162 @@
+import React, { useState, useCallback } from 'react';
+import {
+  Box,
+  Flex,
+  Button,
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  Textarea
+} from '@chakra-ui/react';
+import { useForm } from 'react-hook-form';
+import { postModelDataInput, putModelDataById } from '@/api/model';
+import { useToast } from '@/hooks/useToast';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
+
+export type FormData = { dataId?: string; text: string; q: string };
+
+const InputDataModal = ({
+  onClose,
+  onSuccess,
+  modelId,
+  defaultValues = {
+    text: '',
+    q: ''
+  }
+}: {
+  onClose: () => void;
+  onSuccess: () => void;
+  modelId: string;
+  defaultValues?: FormData;
+}) => {
+  const [importing, setImporting] = useState(false);
+  const { toast } = useToast();
+
+  const { register, handleSubmit } = useForm<FormData>({
+    defaultValues
+  });
+
+  /**
+   * 确认导入新数据
+   */
+  const sureImportData = useCallback(
+    async (e: FormData) => {
+      setImporting(true);
+
+      try {
+        const res = await postModelDataInput({
+          modelId: modelId,
+          data: [
+            {
+              text: e.text,
+              q: {
+                id: nanoid(),
+                text: e.q
+              }
+            }
+          ]
+        });
+
+        toast({
+          title: res === 0 ? '导入数据成功,需要一段时间训练' : '数据导入异常',
+          status: res === 0 ? 'success' : 'warning'
+        });
+        onClose();
+        onSuccess();
+      } catch (err) {
+        console.log(err);
+      }
+      setImporting(false);
+    },
+    [modelId, onClose, onSuccess, toast]
+  );
+
+  const updateData = useCallback(
+    async (e: FormData) => {
+      if (!e.dataId) return;
+      if (e.text === defaultValues.text && e.q === defaultValues.q) return;
+
+      await putModelDataById({
+        dataId: e.dataId,
+        text: e.text,
+        q: e.q === defaultValues.q ? '' : e.q
+      });
+      toast({
+        title: '修改回答成功',
+        status: 'success'
+      });
+      onClose();
+      onSuccess();
+    },
+    [defaultValues.q, onClose, onSuccess, toast]
+  );
+
+  return (
+    <Modal isOpen={true} onClose={onClose} isCentered>
+      <ModalOverlay />
+      <ModalContent
+        m={0}
+        display={'flex'}
+        flexDirection={'column'}
+        h={'90vh'}
+        maxW={'90vw'}
+        position={'relative'}
+      >
+        <ModalHeader>手动导入</ModalHeader>
+        <ModalCloseButton />
+
+        <Box
+          display={['block', 'flex']}
+          flex={'1 0 0'}
+          h={['100%', 0]}
+          overflowY={'auto'}
+          px={6}
+          pb={2}
+        >
+          <Box flex={2} mr={[0, 4]} mb={[4, 0]} h={['230px', '100%']}>
+            <Box h={'30px'}>问题</Box>
+            <Textarea
+              placeholder="相关问题，可以回车输入多个问法, 最多500字"
+              maxLength={500}
+              resize={'none'}
+              h={'calc(100% - 30px)'}
+              {...register(`q`, {
+                required: '相关问题，可以回车输入多个问法'
+              })}
+            />
+          </Box>
+          <Box flex={3} h={['330px', '100%']}>
+            <Box h={'30px'}>知识点</Box>
+            <Textarea
+              placeholder="知识点,最多1000字"
+              maxLength={1000}
+              resize={'none'}
+              h={'calc(100% - 30px)'}
+              {...register(`text`, {
+                required: '知识点'
+              })}
+            />
+          </Box>
+        </Box>
+
+        <Flex px={6} pt={2} pb={4}>
+          <Box flex={1}></Box>
+          <Button variant={'outline'} mr={3} onClick={onClose}>
+            取消
+          </Button>
+          <Button
+            isLoading={importing}
+            onClick={handleSubmit(defaultValues.dataId ? updateData : sureImportData)}
+          >
+            确认导入
+          </Button>
+        </Flex>
+      </ModalContent>
+    </Modal>
+  );
+};
+
+export default InputDataModal;

src/pages/model/detail/components/ModelDataCard.tsx

@@ -0,0 +1,244 @@
+import React, { useCallback, useState } from 'react';
+import {
+  Box,
+  TableContainer,
+  Table,
+  Thead,
+  Tbody,
+  Tr,
+  Th,
+  Td,
+  IconButton,
+  Flex,
+  Button,
+  useDisclosure,
+  Menu,
+  MenuButton,
+  MenuList,
+  MenuItem
+} from '@chakra-ui/react';
+import type { ModelSchema } from '@/types/mongoSchema';
+import type { RedisModelDataItemType } from '@/types/redis';
+import { ModelDataStatusMap } from '@/constants/model';
+import { usePagination } from '@/hooks/usePagination';
+import {
+  getModelDataList,
+  delOneModelData,
+  getModelSplitDataList,
+  getExportDataList
+} from '@/api/model';
+import { DeleteIcon, RepeatIcon, EditIcon } from '@chakra-ui/icons';
+import { useToast } from '@/hooks/useToast';
+import { useLoading } from '@/hooks/useLoading';
+import dynamic from 'next/dynamic';
+import { useMutation, useQuery } from '@tanstack/react-query';
+import type { FormData as InputDataType } from './InputDataModal';
+
+const InputModel = dynamic(() => import('./InputDataModal'));
+const SelectFileModel = dynamic(() => import('./SelectFileModal'));
+const SelectJsonModel = dynamic(() => import('./SelectJsonModal'));
+
+const ModelDataCard = ({ model }: { model: ModelSchema }) => {
+  const { Loading } = useLoading();
+
+  const {
+    data: modelDataList,
+    isLoading,
+    Pagination,
+    total,
+    getData,
+    pageNum
+  } = usePagination<RedisModelDataItemType>({
+    api: getModelDataList,
+    pageSize: 8,
+    params: {
+      modelId: model._id
+    }
+  });
+
+  const [editInputData, setEditInputData] = useState<InputDataType>();
+
+  const {
+    isOpen: isOpenSelectFileModal,
+    onOpen: onOpenSelectFileModal,
+    onClose: onCloseSelectFileModal
+  } = useDisclosure();
+  const {
+    isOpen: isOpenSelectJsonModal,
+    onOpen: onOpenSelectJsonModal,
+    onClose: onCloseSelectJsonModal
+  } = useDisclosure();
+
+  const { data: splitDataList, refetch } = useQuery(['getModelSplitDataList'], () =>
+    getModelSplitDataList(model._id)
+  );
+
+  const refetchData = useCallback(
+    (num = 1) => {
+      getData(num);
+      refetch();
+    },
+    [getData, refetch]
+  );
+
+  // 获取所有的数据，并导出 json
+  const { mutate: onclickExport, isLoading: isLoadingExport } = useMutation({
+    mutationFn: () => getExportDataList(model._id),
+    onSuccess(res) {
+      // 导出为文件
+      const blob = new Blob([res], { type: 'application/json;charset=utf-8' });
+
+      // 创建下载链接
+      const downloadLink = document.createElement('a');
+      downloadLink.href = window.URL.createObjectURL(blob);
+      downloadLink.download = `data.json`;
+
+      // 添加链接到页面并触发下载
+      document.body.appendChild(downloadLink);
+      downloadLink.click();
+      document.body.removeChild(downloadLink);
+    }
+  });
+
+  return (
+    <>
+      <Flex>
+        <Box fontWeight={'bold'} fontSize={'lg'} flex={1} mr={2}>
+          模型数据: {total}组{' '}
+          <Box as={'span'} fontSize={'sm'}>
+            （测试版本）
+          </Box>
+        </Box>
+        <IconButton
+          icon={<RepeatIcon />}
+          aria-label={'refresh'}
+          variant={'outline'}
+          mr={4}
+          size={'sm'}
+          onClick={() => refetchData(pageNum)}
+        />
+        <Button
+          variant={'outline'}
+          mr={2}
+          size={'sm'}
+          isLoading={isLoadingExport}
+          title={'v2.3之前版本的数据无法导出'}
+          onClick={() => onclickExport()}
+        >
+          导出
+        </Button>
+        <Menu>
+          <MenuButton as={Button} size={'sm'}>
+            导入
+          </MenuButton>
+          <MenuList>
+            <MenuItem
+              onClick={() =>
+                setEditInputData({
+                  text: '',
+                  q: ''
+                })
+              }
+            >
+              手动输入
+            </MenuItem>
+            <MenuItem onClick={onOpenSelectFileModal}>文件导入</MenuItem>
+            <MenuItem onClick={onOpenSelectJsonModal}>JSON导入</MenuItem>
+          </MenuList>
+        </Menu>
+      </Flex>
+      {splitDataList && splitDataList.length > 0 && (
+        <Box fontSize={'xs'}>
+          {splitDataList.map((item) => item.textList).flat().length}条数据正在拆分...
+        </Box>
+      )}
+      <Box mt={4}>
+        <TableContainer minH={'500px'}>
+          <Table variant={'simple'}>
+            <Thead>
+              <Tr>
+                <Th>Question</Th>
+                <Th>Text</Th>
+                <Th>Status</Th>
+                <Th>操作</Th>
+              </Tr>
+            </Thead>
+            <Tbody>
+              {modelDataList.map((item) => (
+                <Tr key={item.id}>
+                  <Td>
+                    <Box fontSize={'xs'} w={'100%'} whiteSpace={'pre-wrap'}>
+                      {item.q}
+                    </Box>
+                  </Td>
+                  <Td minW={'200px'}>
+                    <Box w={'100%'} fontSize={'xs'} whiteSpace={'pre-wrap'}>
+                      {item.text}
+                    </Box>
+                  </Td>
+                  <Td>{ModelDataStatusMap[item.status]}</Td>
+                  <Td>
+                    <IconButton
+                      mr={5}
+                      icon={<EditIcon />}
+                      variant={'outline'}
+                      aria-label={'delete'}
+                      size={'sm'}
+                      onClick={() =>
+                        setEditInputData({
+                          dataId: item.id,
+                          q: item.q,
+                          text: item.text
+                        })
+                      }
+                    />
+                    <IconButton
+                      icon={<DeleteIcon />}
+                      variant={'outline'}
+                      colorScheme={'gray'}
+                      aria-label={'delete'}
+                      size={'sm'}
+                      onClick={async () => {
+                        await delOneModelData(item.id);
+                        refetchData(pageNum);
+                      }}
+                    />
+                  </Td>
+                </Tr>
+              ))}
+            </Tbody>
+          </Table>
+        </TableContainer>
+        <Box mt={2} textAlign={'end'}>
+          <Pagination />
+        </Box>
+      </Box>
+
+      <Loading loading={isLoading} fixed={false} />
+      {editInputData !== undefined && (
+        <InputModel
+          modelId={model._id}
+          defaultValues={editInputData}
+          onClose={() => setEditInputData(undefined)}
+          onSuccess={refetchData}
+        />
+      )}
+      {isOpenSelectFileModal && (
+        <SelectFileModel
+          modelId={model._id}
+          onClose={onCloseSelectFileModal}
+          onSuccess={refetchData}
+        />
+      )}
+      {isOpenSelectJsonModal && (
+        <SelectJsonModel
+          modelId={model._id}
+          onClose={onCloseSelectJsonModal}
+          onSuccess={refetchData}
+        />
+      )}
+    </>
+  );
+};
+
+export default ModelDataCard;

src/pages/model/detail/components/ModelEditForm.tsx

@@ -11,13 +11,28 @@ import {
   SliderFilledTrack,
   SliderThumb,
   SliderMark,
-  Tooltip
+  Tooltip,
+  Button
 } from '@chakra-ui/react';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import type { ModelSchema } from '@/types/mongoSchema';
 import { UseFormReturn } from 'react-hook-form';
+import { modelList } from '@/constants/model';
+import { formatPrice } from '@/utils/user';
+import { useConfirm } from '@/hooks/useConfirm';
 
-const ModelEditForm = ({ formHooks }: { formHooks: UseFormReturn<ModelSchema> }) => {
+const ModelEditForm = ({
+  formHooks,
+  canTrain,
+  handleDelModel
+}: {
+  formHooks: UseFormReturn<ModelSchema>;
+  canTrain: boolean;
+  handleDelModel: () => void;
+}) => {
+  const { openConfirm, ConfirmChild } = useConfirm({
+    content: '确认删除该模型?'
+  });
   const { register, setValue, getValues } = formHooks;
   const [refresh, setRefresh] = useState(false);
 
@@ -29,7 +44,7 @@ const ModelEditForm = ({ formHooks }: { formHooks: UseFormReturn<ModelSchema> })
         </Flex>
         <FormControl mt={4}>
           <Flex alignItems={'center'}>
-            <Box flex={'0 0 50px'} w={0}>
+            <Box flex={'0 0 80px'} w={0}>
               名称:
             </Box>
             <Input
@@ -39,7 +54,36 @@ const ModelEditForm = ({ formHooks }: { formHooks: UseFormReturn<ModelSchema> })
             ></Input>
           </Flex>
         </FormControl>
-        <FormControl mt={4}>
+        <Flex alignItems={'center'} mt={4}>
+          <Box flex={'0 0 80px'} w={0}>
+            底层模型:
+          </Box>
+          <Box>{getValues('service.modelName')}</Box>
+        </Flex>
+        <Flex alignItems={'center'} mt={4}>
+          <Box flex={'0 0 80px'} w={0}>
+            价格:
+          </Box>
+          <Box>
+            {formatPrice(
+              modelList.find((item) => item.model === getValues('service.modelName'))?.price || 0,
+              1000
+            )}
+            元/1K tokens(包括上下文和回答)
+          </Box>
+        </Flex>
+        <Flex mt={5} alignItems={'center'}>
+          <Box flex={'0 0 80px'}>删除:</Box>
+          <Button
+            colorScheme={'gray'}
+            variant={'outline'}
+            size={'sm'}
+            onClick={openConfirm(handleDelModel)}
+          >
+            删除模型
+          </Button>
+        </Flex>
+        {/* <FormControl mt={4}>
           <Box mb={1}>介绍:</Box>
           <Textarea
             rows={5}
@@ -47,7 +91,7 @@ const ModelEditForm = ({ formHooks }: { formHooks: UseFormReturn<ModelSchema> })
             {...register('intro')}
             placeholder={'模型的介绍，仅做展示，不影响模型的效果'}
           />
-        </FormControl>
+        </FormControl> */}
       </Card>
       <Card p={4}>
         <Box fontWeight={'bold'}>模型效果</Box>
@@ -64,7 +108,7 @@ const ModelEditForm = ({ formHooks }: { formHooks: UseFormReturn<ModelSchema> })
 
             <Slider
               aria-label="slider-ex-1"
-              min={1}
+              min={0}
               max={10}
               step={1}
               value={getValues('temperature')}
@@ -97,15 +141,17 @@ const ModelEditForm = ({ formHooks }: { formHooks: UseFormReturn<ModelSchema> })
           <Box mb={1}>系统提示词</Box>
           <Textarea
             rows={6}
-            maxLength={500}
+            maxLength={-1}
             {...register('systemPrompt')}
             placeholder={
-              '模型默认的 prompt 词，通过调整该内容，可以生成一个限定范围的模型。\n\n注意，改功能会影响对话的整体朝向！'
+              canTrain
+                ? '训练的模型会根据知识库内容，生成一部分系统提示词，因此在对话时需要消耗更多的 tokens。你仍可以增加一些提示词，让其效果更精确。'
+                : '模型默认的 prompt 词，通过调整该内容，可以生成一个限定范围的模型。\n\n注意，改功能会影响对话的整体朝向！'
             }
           />
         </Box>
       </Card>
-      <Card p={4}>
+      {/* <Card p={4}>
         <Box fontWeight={'bold'}>安全策略</Box>
         <FormControl mt={2}>
           <Flex alignItems={'center'}>
@@ -201,7 +247,8 @@ const ModelEditForm = ({ formHooks }: { formHooks: UseFormReturn<ModelSchema> })
             <Box ml={3}>次</Box>
           </Flex>
         </FormControl>
-      </Card>
+      </Card> */}
+      <ConfirmChild />
     </>
   );
 };

src/pages/model/detail/components/SelectFileModal.tsx

@@ -0,0 +1,175 @@
+import React, { useState, useCallback } from 'react';
+import {
+  Box,
+  Flex,
+  Button,
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody,
+  Input
+} from '@chakra-ui/react';
+import { useToast } from '@/hooks/useToast';
+import { useSelectFile } from '@/hooks/useSelectFile';
+import { customAlphabet } from 'nanoid';
+import { encode } from 'gpt-token-utils';
+import { useConfirm } from '@/hooks/useConfirm';
+import { readTxtContent, readPdfContent, readDocContent } from '@/utils/tools';
+import { useMutation } from '@tanstack/react-query';
+import { postModelDataFileText } from '@/api/model';
+
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
+
+const fileExtension = '.txt,.doc,.docx,.pdf,.md';
+
+const SelectFileModal = ({
+  onClose,
+  onSuccess,
+  modelId
+}: {
+  onClose: () => void;
+  onSuccess: () => void;
+  modelId: string;
+}) => {
+  const [selecting, setSelecting] = useState(false);
+  const { toast } = useToast();
+  const [prompt, setPrompt] = useState('');
+  const { File, onOpen } = useSelectFile({ fileType: fileExtension, multiple: true });
+  const [fileText, setFileText] = useState('');
+  const { openConfirm, ConfirmChild } = useConfirm({
+    content: '确认导入该文件，需要一定时间进行拆解，该任务无法终止！如果余额不足，任务讲被终止。'
+  });
+
+  const onSelectFile = useCallback(
+    async (e: File[]) => {
+      setSelecting(true);
+      try {
+        const fileTexts = (
+          await Promise.all(
+            e.map((file) => {
+              // @ts-ignore
+              const extension = file?.name?.split('.').pop().toLowerCase();
+              switch (extension) {
+                case 'txt':
+                case 'md':
+                  return readTxtContent(file);
+                case 'pdf':
+                  return readPdfContent(file);
+                case 'doc':
+                case 'docx':
+                  return readDocContent(file);
+                default:
+                  return '';
+              }
+            })
+          )
+        )
+          .join('\n')
+          .replace(/\n+/g, '\n');
+        setFileText(fileTexts);
+        console.log(encode(fileTexts));
+      } catch (error: any) {
+        console.log(error);
+        toast({
+          title: typeof error === 'string' ? error : '解析文件失败',
+          status: 'error'
+        });
+      }
+      setSelecting(false);
+    },
+    [setSelecting, toast]
+  );
+
+  const { mutate, isLoading } = useMutation({
+    mutationFn: async () => {
+      if (!fileText) return;
+      await postModelDataFileText({
+        modelId,
+        text: fileText,
+        prompt: `下面是${prompt || '一段长文本'}`
+      });
+      toast({
+        title: '导入数据成功,需要一段拆解和训练',
+        status: 'success'
+      });
+      onClose();
+      onSuccess();
+    },
+    onError() {
+      toast({
+        title: '导入文件失败',
+        status: 'error'
+      });
+    }
+  });
+
+  return (
+    <Modal isOpen={true} onClose={onClose} isCentered>
+      <ModalOverlay />
+      <ModalContent maxW={'min(900px, 90vw)'} m={0} position={'relative'} h={'90vh'}>
+        <ModalHeader>文件导入</ModalHeader>
+        <ModalCloseButton />
+
+        <ModalBody
+          display={'flex'}
+          flexDirection={'column'}
+          p={4}
+          h={'100%'}
+          alignItems={'center'}
+          justifyContent={'center'}
+          fontSize={'sm'}
+        >
+          <Button isLoading={selecting} onClick={onOpen}>
+            选择文件
+          </Button>
+          <Box mt={2} maxW={['100%', '70%']}>
+            支持 {fileExtension} 文件。模型会自动对文本进行 QA 拆分，需要较长训练时间，拆分需要消耗
+            tokens，大约0.04元/1k tokens，请确保账号余额充足。
+          </Box>
+          <Box mt={2}>
+            一共 {fileText.length} 个字，{encode(fileText).length} 个tokens
+          </Box>
+          <Flex w={'100%'} alignItems={'center'} my={4}>
+            <Box flex={'0 0 auto'} mr={2}>
+              下面是
+            </Box>
+            <Input
+              placeholder="提示词，例如: Laf的介绍/关于gpt4的论文/一段长文本"
+              value={prompt}
+              onChange={(e) => setPrompt(e.target.value)}
+              size={'sm'}
+            />
+          </Flex>
+          <Box
+            flex={'1 0 0'}
+            h={0}
+            w={'100%'}
+            overflowY={'auto'}
+            p={2}
+            backgroundColor={'blackAlpha.50'}
+            whiteSpace={'pre-wrap'}
+            fontSize={'xs'}
+          >
+            {fileText}
+          </Box>
+        </ModalBody>
+
+        <Flex px={6} pt={2} pb={4}>
+          <Box flex={1}></Box>
+          <Button variant={'outline'} mr={3} onClick={onClose}>
+            取消
+          </Button>
+          <Button isLoading={isLoading} isDisabled={fileText === ''} onClick={openConfirm(mutate)}>
+            确认导入
+          </Button>
+        </Flex>
+      </ModalContent>
+      <ConfirmChild />
+      <File onSelect={onSelectFile} />
+    </Modal>
+  );
+};
+
+export default SelectFileModal;

src/pages/model/detail/components/SelectJsonModal.tsx

@@ -0,0 +1,145 @@
+import React, { useState, useCallback } from 'react';
+import {
+  Box,
+  Flex,
+  Button,
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody
+} from '@chakra-ui/react';
+import { useToast } from '@/hooks/useToast';
+import { useSelectFile } from '@/hooks/useSelectFile';
+import { useConfirm } from '@/hooks/useConfirm';
+import { readTxtContent } from '@/utils/tools';
+import { useMutation } from '@tanstack/react-query';
+import { postModelDataJsonData } from '@/api/model';
+import Markdown from '@/components/Markdown';
+
+const SelectJsonModal = ({
+  onClose,
+  onSuccess,
+  modelId
+}: {
+  onClose: () => void;
+  onSuccess: () => void;
+  modelId: string;
+}) => {
+  const [selecting, setSelecting] = useState(false);
+  const { toast } = useToast();
+  const { File, onOpen } = useSelectFile({ fileType: '.json', multiple: true });
+  const [fileData, setFileData] = useState<
+    { prompt: string; completion: string; vector?: number[] }[]
+  >([]);
+  const { openConfirm, ConfirmChild } = useConfirm({
+    content: '确认导入该数据集?'
+  });
+
+  const onSelectFile = useCallback(
+    async (e: File[]) => {
+      setSelecting(true);
+      try {
+        const jsonData = (
+          await Promise.all(e.map((item) => readTxtContent(item).then((text) => JSON.parse(text))))
+        ).flat();
+        // check 文件类型
+        for (let i = 0; i < jsonData.length; i++) {
+          if (!jsonData[i]?.prompt || !jsonData[i]?.completion) {
+            throw new Error('缺少 prompt 或 completion');
+          }
+        }
+
+        setFileData(jsonData);
+      } catch (error: any) {
+        console.log(error);
+        toast({
+          title: error?.message || 'JSON文件格式有误',
+          status: 'error'
+        });
+      }
+      setSelecting(false);
+    },
+    [setSelecting, toast]
+  );
+
+  const { mutate, isLoading } = useMutation({
+    mutationFn: async () => {
+      if (!fileData) return;
+      const res = await postModelDataJsonData(modelId, fileData);
+      console.log(res);
+      toast({
+        title: '导入数据成功,需要一段时间训练',
+        status: 'success'
+      });
+      onClose();
+      onSuccess();
+    },
+    onError() {
+      toast({
+        title: '导入文件失败',
+        status: 'error'
+      });
+    }
+  });
+
+  return (
+    <Modal isOpen={true} onClose={onClose} isCentered>
+      <ModalOverlay />
+      <ModalContent maxW={'90vw'} position={'relative'} m={0} h={'90vh'}>
+        <ModalHeader>JSON数据集</ModalHeader>
+        <ModalCloseButton />
+
+        <ModalBody h={'100%'} display={['block', 'flex']} fontSize={'sm'} overflowY={'auto'}>
+          <Box flex={'2 0 0'} w={['100%', 0]} mr={[0, 4]} mb={[4, 0]}>
+            <Markdown
+              source={`接受一个对象数组，每个对象必须包含 prompt 和 completion 格式，可以包含vector。prompt 代表问题，completion 代表回答的内容，可以多个问题对应一个回答，vector 为 prompt 的向量，如果没有讲有系统生成。例如：
+~~~json
+[
+  {
+    "prompt":"sealos是什么?\\n介绍下sealos\\nsealos有什么用",
+    "completion":"sealos是xxxxxx"
+  },
+  {
+    "prompt":"laf是什么?",
+    "completion":"laf是xxxxxx",
+    "vector":[-0.42,-0.4314314,0.43143]
+  }
+]
+~~~`}
+            />
+            <Flex alignItems={'center'}>
+              <Button isLoading={selecting} onClick={onOpen}>
+                选择 JSON 数据集
+              </Button>
+
+              <Box ml={4}>一共 {fileData.length} 组数据</Box>
+            </Flex>
+          </Box>
+          <Box flex={'2 0 0'} h={'100%'} overflow={'auto'} p={2} backgroundColor={'blackAlpha.50'}>
+            {JSON.stringify(fileData)}
+          </Box>
+        </ModalBody>
+
+        <Flex px={6} pt={2} pb={4}>
+          <Box flex={1}></Box>
+          <Button variant={'outline'} mr={3} onClick={onClose}>
+            取消
+          </Button>
+          <Button
+            isLoading={isLoading}
+            isDisabled={fileData.length === 0}
+            onClick={openConfirm(mutate)}
+          >
+            确认导入
+          </Button>
+        </Flex>
+      </ModalContent>
+      <ConfirmChild />
+      <File onSelect={onSelectFile} />
+    </Modal>
+  );
+};
+
+export default SelectJsonModal;

src/pages/model/detail/index.tsx

@@ -1,37 +1,27 @@
 import React, { useCallback, useState, useRef, useMemo, useEffect } from 'react';
 import { useRouter } from 'next/router';
-import {
+import { getModelById, delModelById, putModelTrainingStatus, putModelById } from '@/api/model';
-  getModelById,
-  delModelById,
-  postTrainModel,
-  putModelTrainingStatus,
-  putModelById
-} from '@/api/model';
 import { getChatSiteId } from '@/api/chat';
 import type { ModelSchema } from '@/types/mongoSchema';
 import { Card, Box, Flex, Button, Tag, Grid } from '@chakra-ui/react';
 import { useToast } from '@/hooks/useToast';
-import { useConfirm } from '@/hooks/useConfirm';
 import { useForm } from 'react-hook-form';
 import { formatModelStatus, ModelStatusEnum, modelList, defaultModel } from '@/constants/model';
 import { useGlobalStore } from '@/store/global';
 import { useScreen } from '@/hooks/useScreen';
 import ModelEditForm from './components/ModelEditForm';
-import Icon from '@/components/Iconfont';
 import { useQuery } from '@tanstack/react-query';
 import dynamic from 'next/dynamic';
 
-const Training = dynamic(() => import('./components/Training'));
+const ModelDataCard = dynamic(() => import('./components/ModelDataCard'));
 
 const ModelDetail = ({ modelId }: { modelId: string }) => {
   const { toast } = useToast();
   const router = useRouter();
   const { isPc, media } = useScreen();
   const { setLoading } = useGlobalStore();
-  const { openConfirm, ConfirmChild } = useConfirm({
+
-    content: '确认删除该模型?'
+  // const SelectFileDom = useRef<HTMLInputElement>(null);
-  });
-  const SelectFileDom = useRef<HTMLInputElement>(null);
   const [model, setModel] = useState<ModelSchema>(defaultModel);
   const formHooks = useForm<ModelSchema>({
     defaultValues: model
@@ -39,7 +29,7 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
 
   const canTrain = useMemo(() => {
     const openai = modelList.find((item) => item.model === model?.service.modelName);
-    return openai && openai.trainName;
+    return !!(openai && openai.trainName);
   }, [model]);
 
   /* 加载模型数据 */
@@ -91,34 +81,34 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
   }, [setLoading, model, router]);
 
   /* 上传数据集,触发微调 */
-  const startTraining = useCallback(
+  // const startTraining = useCallback(
-    async (e: React.ChangeEvent<HTMLInputElement>) => {
+  //   async (e: React.ChangeEvent<HTMLInputElement>) => {
-      if (!modelId || !e.target.files || e.target.files?.length === 0) return;
+  //     if (!modelId || !e.target.files || e.target.files?.length === 0) return;
-      setLoading(true);
+  //     setLoading(true);
-      try {
+  //     try {
-        const file = e.target.files[0];
+  //       const file = e.target.files[0];
-        const formData = new FormData();
+  //       const formData = new FormData();
-        formData.append('file', file);
+  //       formData.append('file', file);
-        await postTrainModel(modelId, formData);
+  //       await postTrainModel(modelId, formData);
 
-        toast({
+  //       toast({
-          title: '开始训练，大约需要 30 分钟',
+  //         title: '开始训练...',
-          status: 'success'
+  //         status: 'success'
-        });
+  //       });
 
-        // 重新获取模型
+  //       // 重新获取模型
-        loadModel();
+  //       loadModel();
-      } catch (err) {
+  //     } catch (err: any) {
-        toast({
+  //       toast({
-          title: typeof err === 'string' ? err : '文件格式错误',
+  //         title: err?.message || '上传文件失败',
-          status: 'error'
+  //         status: 'error'
-        });
+  //       });
-        console.log('error->', err);
+  //       console.log('error->', err);
-      }
+  //     }
-      setLoading(false);
+  //     setLoading(false);
-    },
+  //   },
-    [setLoading, loadModel, modelId, toast]
+  //   [setLoading, loadModel, modelId, toast]
-  );
+  // );
 
   /* 点击更新模型状态 */
   const handleClickUpdateStatus = useCallback(async () => {
@@ -126,7 +116,12 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
     setLoading(true);
 
     try {
-      await putModelTrainingStatus(model._id);
+      const res = await putModelTrainingStatus(model._id);
+      typeof res === 'string' &&
+        toast({
+          title: res,
+          status: 'info'
+        });
       loadModel();
     } catch (error: any) {
       console.log('error->', error);
@@ -245,84 +240,33 @@ const ModelDetail = ({ modelId }: { modelId: string }) => {
         )}
       </Card>
       <Grid mt={5} gridTemplateColumns={media('1fr 1fr', '1fr')} gridGap={5}>
-        <ModelEditForm formHooks={formHooks} />
+        <ModelEditForm formHooks={formHooks} handleDelModel={handleDelModel} canTrain={canTrain} />
 
-        {canTrain && (
+        {/* {canTrain && (
           <Card p={4}>
             <Training model={model} />
           </Card>
+        )} */}
+        {canTrain && model._id && (
+          <Card
+            p={4}
+            {...media(
+              {
+                gridColumnStart: 1,
+                gridColumnEnd: 3
+              },
+              {}
             )}
-
-        <Card p={4}>
-          <Box fontWeight={'bold'} fontSize={'lg'}>
-            神奇操作
-          </Box>
-          <Flex mt={5} alignItems={'center'}>
-            <Box flex={'0 0 80px'}>模型微调:</Box>
-            <Button
-              size={'sm'}
-              onClick={() => {
-                SelectFileDom.current?.click();
-              }}
-              title={!canTrain ? '模型不支持微调' : ''}
-              isDisabled={!canTrain}
           >
-              上传数据集
+            <ModelDataCard model={model} />
-            </Button>
-            <Flex
-              as={'a'}
-              href="/TrainingTemplate.jsonl"
-              download
-              ml={5}
-              cursor={'pointer'}
-              alignItems={'center'}
-              color={'blue.500'}
-            >
-              <Icon name={'icon-yunxiazai'} color={'#3182ce'} />
-              下载模板
-            </Flex>
-          </Flex>
-          {/* 提示 */}
-          <Box mt={3} py={3} color={'blackAlpha.600'}>
-            <Box as={'li'} lineHeight={1.9}>
-              可以使用
-              <Box
-                as={'span'}
-                fontWeight={'bold'}
-                textDecoration={'underline'}
-                color={'blackAlpha.800'}
-                mx={2}
-                cursor={'pointer'}
-                onClick={() => router.push('/data/list')}
-              >
-                数据拆分
-              </Box>
-              功能，从任意文本中提取数据集。
-            </Box>
-            <Box as={'li'} lineHeight={1.9}>
-              每行包括一个 prompt 和一个 completion
-            </Box>
-            <Box as={'li'} lineHeight={1.9}>
-              prompt 必须以 {'</s>'} 结尾
-            </Box>
-            <Box as={'li'} lineHeight={1.9}>
-              completion 开头必须有一个空格，必须以 {'</s>'} 结尾
-            </Box>
-          </Box>
-          <Flex mt={5} alignItems={'center'}>
-            <Box flex={'0 0 80px'}>删除模型:</Box>
-            <Button colorScheme={'red'} size={'sm'} onClick={openConfirm(handleDelModel)}>
-              删除模型
-            </Button>
-          </Flex>
           </Card>
+        )}
       </Grid>
 
       {/* 文件选择 */}
-      <Box position={'absolute'} w={0} h={0} overflow={'hidden'}>
+      {/* <Box position={'absolute'} w={0} h={0} overflow={'hidden'}>
         <input ref={SelectFileDom} type="file" accept=".jsonl" onChange={startTraining} />
-      </Box>
+      </Box> */}
-      <ConfirmChild />
     </>
   );
 };

src/pages/model/list/components/ModelTable.tsx

@@ -42,12 +42,12 @@ const ModelTable = ({
       dataIndex: 'status',
       render: (item: ModelSchema) => (
         <Tag
-          colorScheme={formatModelStatus[item.status].colorTheme}
+          colorScheme={formatModelStatus[item.status]?.colorTheme}
           variant="solid"
           px={3}
           size={'md'}
         >
-          {formatModelStatus[item.status].text}
+          {formatModelStatus[item.status]?.text}
         </Tag>
       )
     },

src/service/errorCode.ts

@@ -1,9 +1,10 @@
 export const openaiError: Record<string, string> = {
   context_length_exceeded: '内容超长了，请重置对话',
   Unauthorized: 'API-KEY 不合法',
-  rate_limit_reached: '同时访问用户过多，请稍后再试',
+  rate_limit_reached: 'API被限制，请稍后再试',
   'Bad Request': 'Bad Request~ 可能内容太多了',
-  'Too Many Requests': '请求次数太多了，请慢点~'
+  'Too Many Requests': '请求次数太多了，请慢点~',
+  'Bad Gateway': '网关异常，请重试'
 };
 export const proxyError: Record<string, boolean> = {
   ECONNABORTED: true,

src/service/events/generateAbstract.ts

@@ -0,0 +1,150 @@
+import { DataItem } from '@/service/mongo';
+import { getOpenAIApi } from '@/service/utils/chat';
+import { httpsAgent } from '@/service/utils/tools';
+import { getOpenApiKey } from '../utils/openai';
+import type { ChatCompletionRequestMessage } from 'openai';
+import { DataItemSchema } from '@/types/mongoSchema';
+import { ChatModelNameEnum } from '@/constants/model';
+import { pushSplitDataBill } from '@/service/events/pushBill';
+
+export async function generateAbstract(next = false): Promise<any> {
+  if (global.generatingAbstract && !next) return;
+  global.generatingAbstract = true;
+
+  const systemPrompt: ChatCompletionRequestMessage = {
+    role: 'system',
+    content: `总结助手,我会向你发送一段长文本,请从文本中归纳总结5至15条信息,如果是英文,请增加一条中文的总结,并按以下格式输出: A1:\nA2:\nA3:\n`
+  };
+  let dataItem: DataItemSchema | null = null;
+
+  try {
+    // 找出一个需要生成的 dataItem
+    dataItem = await DataItem.findOne({
+      status: { $ne: 0 },
+      times: { $gt: 0 },
+      type: 'abstract'
+    });
+
+    if (!dataItem) {
+      console.log('没有需要生成 【摘要】 的数据');
+      global.generatingAbstract = false;
+      return;
+    }
+
+    // 更新状态为生成中
+    await DataItem.findByIdAndUpdate(dataItem._id, {
+      status: 2
+    });
+
+    // 获取 openapi Key
+    let userApiKey, systemKey;
+    try {
+      const key = await getOpenApiKey(dataItem.userId, true);
+      userApiKey = key.userApiKey;
+      systemKey = key.systemKey;
+    } catch (error: any) {
+      if (error?.code === 501) {
+        // 余额不够了, 把用户所有记录改成闲置
+        await DataItem.updateMany({
+          userId: dataItem.userId,
+          status: 0
+        });
+      }
+
+      throw new Error('获取 openai key 失败');
+    }
+
+    console.log('正在生成一组摘要, ID:', dataItem._id);
+
+    const startTime = Date.now();
+
+    // 获取 openai 请求实例
+    const chatAPI = getOpenAIApi(userApiKey || systemKey);
+    // 请求 chatgpt 获取摘要
+    const abstractResponse = await chatAPI.createChatCompletion(
+      {
+        model: ChatModelNameEnum.GPT35,
+        temperature: 0.8,
+        n: 1,
+        messages: [
+          systemPrompt,
+          {
+            role: 'user',
+            content: dataItem?.text || ''
+          }
+        ]
+      },
+      {
+        timeout: 120000,
+        httpsAgent
+      }
+    );
+
+    // 提取摘要内容
+    const rawContent: string = abstractResponse?.data.choices[0].message?.content || '';
+    // 从 content 中提取摘要内容
+    const splitContents = splitText(rawContent);
+
+    // 插入数据库，并修改状态
+    await DataItem.findByIdAndUpdate(dataItem._id, {
+      status: 0,
+      $push: {
+        rawResponse: rawContent,
+        result: {
+          $each: splitContents
+        }
+      }
+    });
+
+    console.log(
+      `生成摘要成功，time: ${(Date.now() - startTime) / 1000}s`,
+      `摘要匹配数量: ${splitContents.length}`
+    );
+    // 计费
+    pushSplitDataBill({
+      isPay: !userApiKey && splitContents.length > 0,
+      userId: dataItem.userId,
+      type: 'abstract',
+      text: systemPrompt.content + dataItem.text + rawContent
+    });
+  } catch (error: any) {
+    console.log('error: 生成摘要错误', dataItem?._id);
+    console.log('response:', error);
+    if (dataItem?._id) {
+      await DataItem.findByIdAndUpdate(dataItem._id, {
+        status: dataItem.times > 1 ? 1 : 0, // 还有重试次数则可以继续进行
+        $inc: {
+          // 剩余尝试次数-1
+          times: -1
+        }
+      });
+    }
+  }
+
+  generateAbstract(true);
+}
+
+/**
+ * 检查文本是否按格式返回
+ */
+function splitText(text: string) {
+  const regex = /A\d+:(\s*)(.*?)\s*(?=A\d+:|$)/gs;
+  const matches = text.matchAll(regex); // 获取所有匹配到的结果
+
+  const result = []; // 存储最终的结果
+  for (const match of matches) {
+    if (match[2]) {
+      result.push({
+        abstract: match[2] as string
+      });
+    }
+  }
+
+  if (result.length === 0) {
+    result.push({
+      abstract: text
+    });
+  }
+
+  return result;
+}

src/service/events/generateQA.ts

@@ -1,26 +1,25 @@
-import { DataItem } from '@/service/mongo';
+import { SplitData } from '@/service/mongo';
 import { getOpenAIApi } from '@/service/utils/chat';
-import { httpsAgent, getOpenApiKey } from '@/service/utils/tools';
+import { httpsAgent } from '@/service/utils/tools';
+import { getOpenApiKey } from '../utils/openai';
 import type { ChatCompletionRequestMessage } from 'openai';
-import { DataItemSchema } from '@/types/mongoSchema';
 import { ChatModelNameEnum } from '@/constants/model';
 import { pushSplitDataBill } from '@/service/events/pushBill';
+import { generateVector } from './generateVector';
+import { connectRedis } from '../redis';
+import { VecModelDataPrefix } from '@/constants/redis';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
 
 export async function generateQA(next = false): Promise<any> {
   if (global.generatingQA && !next) return;
   global.generatingQA = true;
 
-  const systemPrompt: ChatCompletionRequestMessage = {
-    role: 'system',
-    content: `总结助手。我会向你发送一段长文本,请从中总结出5至15个问题和答案,答案请尽量详细,请按以下格式返回: "Q1:"\n"A1:"\n"Q2:"\n"A2:"\n`
-  };
-  let dataItem: DataItemSchema | null = null;
-
   try {
+    const redis = await connectRedis();
     // 找出一个需要生成的 dataItem
-    dataItem = await DataItem.findOne({
+    const dataItem = await SplitData.findOne({
-      status: { $ne: 0 },
+      textList: { $exists: true, $ne: [] }
-      times: { $gt: 0 }
     });
 
     if (!dataItem) {
@@ -29,101 +28,123 @@ export async function generateQA(next = false): Promise<any> {
       return;
     }
 
-    // 更新状态为生成中
+    // 源文本
-    await DataItem.findByIdAndUpdate(dataItem._id, {
+    const text = dataItem.textList[dataItem.textList.length - 1];
-      status: 2
+    if (!text) {
-    });
+      await SplitData.findByIdAndUpdate(dataItem._id, { $pop: { textList: 1 } }); // 弹出无效文本
+      throw new Error('无文本');
+    }
 
     // 获取 openapi Key
     let userApiKey, systemKey;
     try {
-      const key = await getOpenApiKey(dataItem.userId);
+      const key = await getOpenApiKey(dataItem.userId, true);
       userApiKey = key.userApiKey;
       systemKey = key.systemKey;
-    } catch (error) {
+    } catch (error: any) {
-      // 余额不够了, 把用户所有记录改成闲置
+      if (error?.code === 501) {
-      await DataItem.updateMany({
+        // 余额不够了, 清空该记录
-        userId: dataItem.userId,
+        await SplitData.findByIdAndUpdate(dataItem._id, {
-        status: 0
+          textList: [],
+          errorText: error.message
         });
+        throw new Error('账号余额不足');
+      }
+
       throw new Error('获取 openai key 失败');
     }
 
-    console.log('正在生成一个QA, ID:', dataItem._id, 'temperature: ', dataItem.temperature / 100);
+    console.log('正在生成一组QA, ID:', dataItem._id);
 
     const startTime = Date.now();
 
     // 获取 openai 请求实例
     const chatAPI = getOpenAIApi(userApiKey || systemKey);
+    const systemPrompt: ChatCompletionRequestMessage = {
+      role: 'system',
+      content: `${
+        dataItem.prompt || '下面是一段长文本'
+      },请从中提取出5至30个问题和答案,并按以下格式返回: Q1:\nA1:\nQ2:\nA2:\n`
+    };
+
     // 请求 chatgpt 获取回答
-    const response = await chatAPI.createChatCompletion(
+    const response = await chatAPI
+      .createChatCompletion(
         {
           model: ChatModelNameEnum.GPT35,
-        temperature: dataItem.temperature / 100,
+          temperature: 0.8,
           n: 1,
           messages: [
             systemPrompt,
             {
               role: 'user',
-            content: dataItem.text
+              content: text
             }
           ]
         },
         {
-        timeout: 120000,
+          timeout: 180000,
           httpsAgent
         }
-    );
+      )
-    const content = response.data.choices[0].message?.content;
+      .then((res) => ({
-    // 从 content 中提取 QA
+        rawContent: res?.data.choices[0].message?.content || '', // chatgpt原本的回复
-    const splitResponse = splitText(content || '');
+        result: splitText(res?.data.choices[0].message?.content || '') // 格式化后的QA对
-    // 插入数据库，并修改状态
+      }));
-    await DataItem.findByIdAndUpdate(dataItem._id, {
+
-      status: dataItem.temperature >= 90 ? 0 : 1, // 需要生成 4 组内容。0,0.3,0.6,0.9
+    await Promise.allSettled([
-      temperature: dataItem.temperature >= 90 ? dataItem.temperature : dataItem.temperature + 30,
+      SplitData.findByIdAndUpdate(dataItem._id, { $pop: { textList: 1 } }), // 弹出已经拆分的文本
-      $push: {
+      ...response.result.map((item) => {
-        rawResponse: content,
+        // 插入 redis
-        result: {
+        return redis.sendCommand([
-          $each: splitResponse
+          'HMSET',
-        }
+          `${VecModelDataPrefix}:${nanoid()}`,
-      }
+          'userId',
-    });
+          String(dataItem.userId),
-    // 计费
+          'modelId',
-    !userApiKey &&
+          String(dataItem.modelId),
-      splitResponse.length > 0 &&
+          'q',
-      pushSplitDataBill({
+          item.q,
-        userId: dataItem.userId,
+          'text',
-        text: systemPrompt.content + dataItem.text + content
+          item.a,
-      });
+          'status',
+          'waiting'
+        ]);
+      })
+    ]);
+
     console.log(
       '生成QA成功，time:',
       `${(Date.now() - startTime) / 1000}s`,
       'QA数量：',
-      splitResponse.length
+      response.result.length
     );
-  } catch (error: any) {
+
-    console.log('error: 生成QA错误', dataItem?._id);
+    // 计费
-    console.log('response:', error?.response);
+    pushSplitDataBill({
-    // 重置状态
+      isPay: !userApiKey && response.result.length > 0,
-    if (dataItem?._id) {
+      userId: dataItem.userId,
-      await DataItem.findByIdAndUpdate(dataItem._id, {
+      type: 'QA',
-        status: dataItem.times > 0 ? 1 : 0, // 还有重试次数则可以继续进行
+      text: systemPrompt.content + text + response.rawContent
-        $inc: {
-          // 剩余尝试次数-1
-          times: -1
-        }
     });
-    }
-  }
 
     generateQA(true);
+    generateVector(true);
+  } catch (error: any) {
+    console.log(error);
+    console.log('生成QA错误:', error?.response);
+
+    setTimeout(() => {
+      generateQA(true);
+    }, 5000);
+  }
 }
 
 /**
  * 检查文本是否按格式返回
  */
 function splitText(text: string) {
-  const regex = /Q\d+:(\s*)(.*)(\s*)A\d+:(\s*)(.*)(\s*)/g; // 匹配Q和A的正则表达式
+  const regex = /Q\d+:(\s*)(.*)(\s*)A\d+:(\s*)([\s\S]*?)(?=Q|$)/g; // 匹配Q和A的正则表达式
   const matches = text.matchAll(regex); // 获取所有匹配到的结果
 
   const result = []; // 存储最终的结果
@@ -131,7 +152,11 @@ function splitText(text: string) {
     const q = match[2];
     const a = match[5];
     if (q && a) {
-      result.push({ q, a }); // 如果Q和A都存在，就将其添加到结果中
+      // 如果Q和A都存在，就将其添加到结果中
+      result.push({
+        q,
+        a: a.trim().replace(/\n\s*/g, '\n')
+      });
     }
   }
 

src/service/events/generateVector.ts

@@ -0,0 +1,85 @@
+import { getOpenAIApi } from '@/service/utils/chat';
+import { httpsAgent } from '@/service/utils/tools';
+import { connectRedis } from '../redis';
+import { VecModelDataIdx } from '@/constants/redis';
+import { vectorToBuffer } from '@/utils/tools';
+import { ModelDataStatusEnum } from '@/constants/redis';
+import { openaiCreateEmbedding, getOpenApiKey } from '../utils/openai';
+
+export async function generateVector(next = false): Promise<any> {
+  if (global.generatingVector && !next) return;
+  global.generatingVector = true;
+
+  try {
+    const redis = await connectRedis();
+
+    // 从找出一个 status = waiting 的数据
+    const searchRes = await redis.ft.search(
+      VecModelDataIdx,
+      `@status:{${ModelDataStatusEnum.waiting}}`,
+      {
+        RETURN: ['q', 'userId'],
+        LIMIT: {
+          from: 0,
+          size: 1
+        }
+      }
+    );
+
+    if (searchRes.total === 0) {
+      console.log('没有需要生成 【向量】 的数据');
+      global.generatingVector = false;
+      return;
+    }
+
+    const dataItem: { id: string; q: string; userId: string } = {
+      id: searchRes.documents[0].id,
+      q: String(searchRes.documents[0]?.value?.q || ''),
+      userId: String(searchRes.documents[0]?.value?.userId || '')
+    };
+
+    // 获取 openapi Key
+    const { userApiKey, systemKey } = await getOpenApiKey(dataItem.userId);
+
+    // 生成词向量
+    const { vector } = await openaiCreateEmbedding({
+      text: dataItem.q,
+      userId: dataItem.userId,
+      isPay: !userApiKey,
+      apiKey: userApiKey || systemKey
+    });
+
+    // 更新 redis 向量和状态数据
+    await redis.sendCommand([
+      'HMSET',
+      dataItem.id,
+      'vector',
+      vectorToBuffer(vector),
+      'rawVector',
+      JSON.stringify(vector),
+      'status',
+      ModelDataStatusEnum.ready
+    ]);
+
+    console.log(`生成向量成功: ${dataItem.id}`);
+
+    setTimeout(() => {
+      generateVector(true);
+    }, 2000);
+  } catch (error: any) {
+    console.log('error: 生成向量错误', error?.response?.statusText);
+    !error?.response && console.log(error);
+
+    if (error?.response?.statusText === 'Too Many Requests') {
+      console.log('生成向量次数限制，1分钟后尝试');
+      // 限制次数，1分钟后再试
+      setTimeout(() => {
+        generateVector(true);
+      }, 60000);
+    }
+
+    setTimeout(() => {
+      generateVector(true);
+    }, 3000);
+  }
+}

src/service/events/pushBill.ts

@@ -2,36 +2,39 @@ import { connectToDatabase, Bill, User } from '../mongo';
 import { modelList, ChatModelNameEnum } from '@/constants/model';
 import { encode } from 'gpt-token-utils';
 import { formatPrice } from '@/utils/user';
+import { BillTypeEnum } from '@/constants/user';
+import type { DataType } from '@/types/data';
 
 export const pushChatBill = async ({
+  isPay,
   modelName,
   userId,
   chatId,
   text
 }: {
+  isPay: boolean;
   modelName: string;
   userId: string;
   chatId: string;
   text: string;
 }) => {
-  await connectToDatabase();
-
   let billId;
 
   try {
-    // 获取模型单价格
-    const modelItem = modelList.find((item) => item.model === modelName);
-    const unitPrice = modelItem?.price || 5;
-
     // 计算 token 数量
     const tokens = encode(text);
 
+    console.log(`chat generate success. text len: ${text.length}. token len: ${tokens.length}`);
+
+    if (isPay) {
+      await connectToDatabase();
+
+      // 获取模型单价格
+      const modelItem = modelList.find((item) => item.model === modelName);
       // 计算价格
+      const unitPrice = modelItem?.price || 5;
       const price = unitPrice * tokens.length;
-    console.log('chat bill');
+      console.log(`unit price: ${unitPrice}, price: ${formatPrice(price)}元`);
-    console.log('token len:', tokens.length);
-    console.log('text len: ', text.length);
-    console.log('price: ', `${formatPrice(price)}元`);
 
       try {
         // 插入 Bill 记录
@@ -54,36 +57,49 @@ export const pushChatBill = async ({
         console.log('创建账单失败:', error);
         billId && Bill.findByIdAndDelete(billId);
       }
+    }
   } catch (error) {
     console.log(error);
   }
 };
 
-export const pushSplitDataBill = async ({ userId, text }: { userId: string; text: string }) => {
+export const pushSplitDataBill = async ({
+  isPay,
+  userId,
+  text,
+  type
+}: {
+  isPay: boolean;
+  userId: string;
+  text: string;
+  type: DataType;
+}) => {
   await connectToDatabase();
 
   let billId;
 
   try {
-    // 获取模型单价格, 都是用 gpt35 拆分
-    const modelItem = modelList.find((item) => item.model === ChatModelNameEnum.GPT35);
-    const unitPrice = modelItem?.price || 5;
-
     // 计算 token 数量
     const tokens = encode(text);
 
+    console.log(
+      `splitData generate success. text len: ${text.length}. token len: ${tokens.length}`
+    );
+
+    if (isPay) {
+      try {
+        // 获取模型单价格, 都是用 gpt35 拆分
+        const modelItem = modelList.find((item) => item.model === ChatModelNameEnum.GPT35);
+        const unitPrice = modelItem?.price || 5;
         // 计算价格
         const price = unitPrice * tokens.length;
-    console.log('splitData bill');
-    console.log('token len:', tokens.length);
-    console.log('text len: ', text.length);
-    console.log('price: ', `${formatPrice(price)}元`);
 
-    try {
+        console.log(`price: ${formatPrice(price)}元`);
+
         // 插入 Bill 记录
         const res = await Bill.create({
           userId,
-        type: 'splitData',
+          type,
           modelName: ChatModelNameEnum.GPT35,
           textLen: text.length,
           tokenLen: tokens.length,
@@ -99,6 +115,59 @@ export const pushSplitDataBill = async ({ userId, text }: { userId: string; text
         console.log('创建账单失败:', error);
         billId && Bill.findByIdAndDelete(billId);
       }
+    }
+  } catch (error) {
+    console.log(error);
+  }
+};
+
+export const pushGenerateVectorBill = async ({
+  isPay,
+  userId,
+  text
+}: {
+  isPay: boolean;
+  userId: string;
+  text: string;
+}) => {
+  await connectToDatabase();
+
+  let billId;
+
+  try {
+    // 计算 token 数量
+    const tokens = encode(text);
+
+    console.log(`vector generate success. text len: ${text.length}. token len: ${tokens.length}`);
+
+    if (isPay) {
+      try {
+        const unitPrice = 1;
+        // 计算价格
+        const price = unitPrice * tokens.length;
+
+        console.log(`price: ${formatPrice(price)}元`);
+
+        // 插入 Bill 记录
+        const res = await Bill.create({
+          userId,
+          type: BillTypeEnum.vector,
+          modelName: ChatModelNameEnum.VECTOR,
+          textLen: text.length,
+          tokenLen: tokens.length,
+          price
+        });
+        billId = res._id;
+
+        // 账号扣费
+        await User.findByIdAndUpdate(userId, {
+          $inc: { balance: -price }
+        });
+      } catch (error) {
+        console.log('创建账单失败:', error);
+        billId && Bill.findByIdAndDelete(billId);
+      }
+    }
   } catch (error) {
     console.log(error);
   }

src/service/models/bill.ts

@@ -1,6 +1,7 @@
 import { Schema, model, models, Model } from 'mongoose';
 import { modelList } from '@/constants/model';
 import { BillSchema as BillType } from '@/types/mongoSchema';
+import { BillTypeMap } from '@/constants/user';
 
 const BillSchema = new Schema({
   userId: {
@@ -10,12 +11,12 @@ const BillSchema = new Schema({
   },
   type: {
     type: String,
-    enum: ['chat', 'splitData', 'return'],
+    enum: Object.keys(BillTypeMap),
     required: true
   },
   modelName: {
     type: String,
-    enum: modelList.map((item) => item.model),
+    enum: [...modelList.map((item) => item.model), 'text-embedding-ada-002'],
     required: true
   },
   chatId: {

src/service/models/chat.ts

@@ -23,8 +23,8 @@ const ChatSchema = new Schema({
     required: true
   },
   updateTime: {
-    type: Number,
+    type: Date,
-    required: true
+    default: () => new Date()
   },
   isShare: {
     type: Boolean,
@@ -41,6 +41,10 @@ const ChatSchema = new Schema({
         value: {
           type: String,
           required: true
+        },
+        deleted: {
+          type: Boolean,
+          default: false
         }
       }
     ],

src/service/models/data.ts

@@ -1,5 +1,6 @@
 import { Schema, model, models, Model } from 'mongoose';
-import { DataItemSchema as Datatype } from '@/types/mongoSchema';
+import { DataSchema as Datatype } from '@/types/mongoSchema';
+import { DataTypeTextMap } from '@/constants/data';
 
 const DataSchema = new Schema({
   userId: {
@@ -15,6 +16,11 @@ const DataSchema = new Schema({
     type: Date,
     default: () => new Date()
   },
+  type: {
+    type: String,
+    required: true,
+    enum: Object.keys(DataTypeTextMap)
+  },
   isDeleted: {
     type: Boolean,
     default: false

src/service/models/dataItem.ts

@@ -1,5 +1,6 @@
 import type { DataItemSchema as DataItemType } from '@/types/mongoSchema';
 import { Schema, model, models, Model } from 'mongoose';
+import { DataTypeTextMap } from '@/constants/data';
 
 const DataItemSchema = new Schema({
   userId: {
@@ -12,19 +13,23 @@ const DataItemSchema = new Schema({
     ref: 'data',
     required: true
   },
+  type: {
+    type: String,
+    required: true,
+    enum: Object.keys(DataTypeTextMap)
+  },
   times: {
+    // 剩余重试次数
     type: Number,
     default: 3
   },
   text: {
+    // 文本内容
     type: String,
     required: true
   },
-  temperature: {
-    type: Number,
-    required: true
-  },
   rawResponse: {
+    // 原始拆分结果
     type: [String],
     default: []
   },
@@ -33,11 +38,21 @@ const DataItemSchema = new Schema({
       {
         q: {
           type: String,
-          required: true
+          default: ''
         },
         a: {
           type: String,
-          required: true
+          default: ''
+        },
+        abstract: {
+          // 摘要
+          type: String,
+          default: ''
+        },
+        abstractVector: {
+          // 摘要对应的向量
+          type: [Number],
+          default: []
         }
       }
     ],

src/service/models/model.ts

@@ -50,9 +50,9 @@ const ModelSchema = new Schema({
       enum: ['openai']
     },
     trainId: {
-      // 训练时需要的 ID
+      // 训练时需要的 ID， 不能训练的模型没有这个值。
       type: String,
-      required: true
+      required: false
     },
     chatModel: {
       // 聊天时使用的模型

src/service/models/splitData.ts

@@ -0,0 +1,36 @@
+/* 模型的知识库 */
+import { Schema, model, models, Model as MongoModel } from 'mongoose';
+import { ModelSplitDataSchema as SplitDataType } from '@/types/mongoSchema';
+
+const SplitDataSchema = new Schema({
+  userId: {
+    type: Schema.Types.ObjectId,
+    ref: 'user',
+    required: true
+  },
+  prompt: {
+    // 拆分时的提示词
+    type: String,
+    required: true
+  },
+  modelId: {
+    type: Schema.Types.ObjectId,
+    ref: 'model',
+    required: true
+  },
+  rawText: {
+    type: String,
+    required: true
+  },
+  textList: {
+    type: [String],
+    default: []
+  },
+  errorText: {
+    type: String,
+    default: ''
+  }
+});
+
+export const SplitData: MongoModel<SplitDataType> =
+  models['splitData'] || model('splitData', SplitDataSchema);

src/service/mongo.ts

@@ -1,5 +1,8 @@
 import mongoose from 'mongoose';
 import { generateQA } from './events/generateQA';
+import { generateAbstract } from './events/generateAbstract';
+import { generateVector } from './events/generateVector';
+
 /**
  * 连接 MongoDB 数据库
  */
@@ -14,7 +17,7 @@ export async function connectToDatabase(): Promise<void> {
     mongoose.set('strictQuery', true);
     global.mongodb = await mongoose.connect(process.env.MONGODB_URI as string, {
       bufferCommands: true,
-      dbName: 'doc_gpt',
+      dbName: process.env.NODE_ENV === 'development' ? 'doc_gpt_test' : 'doc_gpt',
       maxPoolSize: 5,
       minPoolSize: 1,
       maxConnecting: 5
@@ -24,8 +27,9 @@ export async function connectToDatabase(): Promise<void> {
     global.mongodb = null;
   }
 
-  // 递归 QA 生成
   generateQA();
+  // generateAbstract();
+  generateVector();
 }
 
 export * from './models/authCode';
@@ -37,3 +41,4 @@ export * from './models/bill';
 export * from './models/pay';
 export * from './models/data';
 export * from './models/dataItem';
+export * from './models/splitData';

src/service/redis.ts

@@ -0,0 +1,45 @@
+import { createClient } from 'redis';
+import { customAlphabet } from 'nanoid';
+const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 10);
+
+export const connectRedis = async () => {
+  // 断开了，重连
+  if (global.redisClient && !global.redisClient.isOpen) {
+    await global.redisClient.disconnect();
+  } else if (global.redisClient) {
+    // 没断开，不再连接
+    return global.redisClient;
+  }
+
+  try {
+    global.redisClient = createClient({
+      url: process.env.REDIS_URL
+    });
+
+    global.redisClient.on('error', (err) => {
+      console.log('Redis Client Error', err);
+      global.redisClient = null;
+    });
+    global.redisClient.on('end', () => {
+      global.redisClient = null;
+    });
+    global.redisClient.on('ready', () => {
+      console.log('redis connected');
+    });
+
+    await global.redisClient.connect();
+
+    // 1 - 测试库，0 - 正式
+    await global.redisClient.SELECT(0);
+
+    return global.redisClient;
+  } catch (error) {
+    console.log(error, '==');
+    global.redisClient = null;
+    return Promise.reject('redis 连接失败');
+  }
+};
+
+export const getKey = (prefix = '') => {
+  return `${prefix}:${nanoid()}`;
+};

src/service/response.ts

@@ -32,7 +32,6 @@ export const jsonRes = <T = any>(
     console.log('code:', error.code);
     console.log('statusText:', error?.response?.statusText);
     console.log('msg:', msg);
-    error?.response && console.log('chat err:', error?.response);
   }
 
   res.json({

src/service/utils/chat.ts

@@ -1,8 +1,8 @@
 import { Configuration, OpenAIApi } from 'openai';
 import { Chat } from '../mongo';
 import type { ChatPopulate } from '@/types/mongoSchema';
-import { formatPrice } from '@/utils/user';
 import { authToken } from './tools';
+import { getOpenApiKey } from './openai';
 
 export const getOpenAIApi = (apiKey: string) => {
   const configuration = new Configuration({
@@ -14,46 +14,39 @@ export const getOpenAIApi = (apiKey: string) => {
 
 export const authChat = async (chatId: string, authorization?: string) => {
   // 获取 chat 数据
-  const chat = await Chat.findById<ChatPopulate>(chatId)
+  const chat = await Chat.findById<ChatPopulate>(chatId).populate({
-    .populate({
     path: 'modelId',
     options: {
       strictPopulate: false
     }
-    })
-    .populate({
-      path: 'userId',
-      options: {
-        strictPopulate: false
-      }
   });
 
   if (!chat || !chat.modelId || !chat.userId) {
     return Promise.reject('模型不存在');
   }
 
-  // 安全校验
+  // 凭证校验
-  if (chat.loadAmount === 0 || chat.expiredTime <= Date.now()) {
+  if (!chat.isShare) {
+    const userId = await authToken(authorization);
+    if (userId !== String(chat.userId._id)) {
+      return Promise.reject('无权使用该对话');
+    }
+  } else if (chat.loadAmount === 0 || chat.expiredTime <= Date.now()) {
     return Promise.reject('聊天框已过期');
   }
 
-  // 分享校验
-  if (!chat.isShare) {
-    await authToken(authorization);
-  }
-
   // 获取 user 的 apiKey
-  const user = chat.userId;
+  const { user, userApiKey, systemKey } = await getOpenApiKey(
+    chat.userId as unknown as string,
+    false
+  );
 
-  const userApiKey = user.accounts?.find((item: any) => item.type === 'openai')?.value;
+  // filter 掉被 deleted 的内容
-
+  chat.content = chat.content.filter((item) => item.deleted !== true);
-  if (!userApiKey && formatPrice(user.balance) <= 0) {
-    return Promise.reject('该账号余额不足');
-  }
 
   return {
     userApiKey,
-    systemKey: process.env.OPENAIKEY as string,
+    systemKey,
     chat,
     userId: user._id
   };

src/service/utils/openai.ts

@@ -0,0 +1,132 @@
+import axios from 'axios';
+import { getOpenAIApi } from '@/service/utils/chat';
+import { httpsAgent } from './tools';
+import { User } from '../models/user';
+import { formatPrice } from '@/utils/user';
+import { ChatModelNameEnum } from '@/constants/model';
+import { pushGenerateVectorBill } from '../events/pushBill';
+
+/* 判断 apikey 是否还有余额 */
+export const checkKeyGrant = async (apiKey: string) => {
+  const grant = await axios.get('https://api.openai.com/dashboard/billing/credit_grants', {
+    headers: {
+      Authorization: `Bearer ${apiKey}`
+    },
+    httpsAgent
+  });
+  if (grant.data?.total_available <= 0.2) {
+    return false;
+  }
+  return true;
+};
+
+/* 获取用户 api 的 openai 信息 */
+export const getUserApiOpenai = async (userId: string) => {
+  const user = await User.findById(userId);
+
+  const userApiKey = user?.accounts?.find((item: any) => item.type === 'openai')?.value;
+
+  if (!userApiKey) {
+    return Promise.reject('缺少ApiKey, 无法请求');
+  }
+
+  // 余额校验
+  const hasGrant = await checkKeyGrant(userApiKey);
+  if (!hasGrant) {
+    return Promise.reject({
+      code: 501,
+      message: 'API 余额不足'
+    });
+  }
+
+  return {
+    user,
+    openai: getOpenAIApi(userApiKey),
+    apiKey: userApiKey
+  };
+};
+
+/* 获取 open api key，如果用户没有自己的key，就用平台的，用平台记得加账单 */
+export const getOpenApiKey = async (userId: string, checkGrant = false) => {
+  const user = await User.findById(userId);
+  if (!user) {
+    return Promise.reject('找不到用户');
+  }
+
+  const userApiKey = user?.accounts?.find((item: any) => item.type === 'openai')?.value;
+
+  // 有自己的key
+  if (userApiKey) {
+    // api 余额校验
+    if (checkGrant) {
+      const hasGrant = await checkKeyGrant(userApiKey);
+      if (!hasGrant) {
+        return Promise.reject({
+          code: 501,
+          message: 'API 余额不足'
+        });
+      }
+    }
+
+    return {
+      user,
+      userApiKey,
+      systemKey: ''
+    };
+  }
+
+  // 平台账号余额校验
+  if (formatPrice(user.balance) <= 0) {
+    return Promise.reject({
+      code: 501,
+      message: '账号余额不足'
+    });
+  }
+
+  return {
+    user,
+    userApiKey: '',
+    systemKey: process.env.OPENAIKEY as string
+  };
+};
+
+/* 获取向量 */
+export const openaiCreateEmbedding = async ({
+  isPay,
+  userId,
+  apiKey,
+  text
+}: {
+  isPay: boolean;
+  userId: string;
+  apiKey: string;
+  text: string;
+}) => {
+  // 获取 chatAPI
+  const chatAPI = getOpenAIApi(apiKey);
+
+  // 把输入的内容转成向量
+  const vector = await chatAPI
+    .createEmbedding(
+      {
+        model: ChatModelNameEnum.VECTOR,
+        input: text
+      },
+      {
+        timeout: 60000,
+        httpsAgent
+      }
+    )
+    .then((res) => res?.data?.data?.[0]?.embedding || []);
+
+  pushGenerateVectorBill({
+    isPay,
+    userId,
+    text
+  });
+
+  return {
+    vector,
+    chatAPI
+  };
+};

src/service/utils/tools.ts

@@ -1,9 +1,8 @@
 import crypto from 'crypto';
 import jwt from 'jsonwebtoken';
-import { User } from '../models/user';
 import tunnel from 'tunnel';
-import type { UserModelSchema } from '@/types/mongoSchema';
+import { ChatItemType } from '@/types/chat';
-import { formatPrice } from '@/utils/user';
+import { encode } from 'gpt-token-utils';
 
 /* 密码加密 */
 export const hashPassword = (psw: string) => {
@@ -42,45 +41,6 @@ export const authToken = (token?: string): Promise<string> => {
   });
 };
 
-/* 获取用户的 openai APIkey */
-export const getUserOpenaiKey = async (userId: string) => {
-  const user = await User.findById(userId);
-
-  const userApiKey = user?.accounts?.find((item: any) => item.type === 'openai')?.value;
-  if (!userApiKey) {
-    return Promise.reject('缺少ApiKey, 无法请求');
-  }
-
-  return Promise.resolve(userApiKey as string);
-};
-
-/* 获取key，如果没有就用平台的，用平台记得加账单 */
-export const getOpenApiKey = async (userId: string) => {
-  const user = await User.findById<UserModelSchema>(userId);
-
-  if (!user) return Promise.reject('用户不存在');
-
-  const userApiKey = user.accounts?.find((item: any) => item.type === 'openai')?.value;
-
-  // 有自己的key， 直接使用
-  if (userApiKey) {
-    return {
-      userApiKey: await getUserOpenaiKey(userId),
-      systemKey: ''
-    };
-  }
-
-  // 余额校验
-  if (formatPrice(user.balance) <= 0) {
-    return Promise.reject('该账号余额不足');
-  }
-
-  return {
-    userApiKey: '',
-    systemKey: process.env.OPENAIKEY as string
-  };
-};
-
 /* 代理 */
 export const httpsAgent =
   process.env.AXIOS_PROXY_HOST && process.env.AXIOS_PROXY_PORT
@@ -91,3 +51,47 @@ export const httpsAgent =
         }
       })
     : undefined;
+
+/* tokens 截断 */
+export const openaiChatFilter = (prompts: ChatItemType[], maxTokens: number) => {
+  let res: ChatItemType[] = [];
+
+  let systemPrompt: ChatItemType | null = null;
+
+  //  System 词保留
+  if (prompts[0]?.obj === 'SYSTEM') {
+    systemPrompt = prompts.shift() as ChatItemType;
+    maxTokens -= encode(prompts[0].value).length;
+  }
+
+  // 从后往前截取
+  for (let i = prompts.length - 1; i >= 0; i--) {
+    const tokens = encode(prompts[i].value).length;
+    if (maxTokens >= tokens) {
+      res.unshift(prompts[i]);
+      maxTokens -= tokens;
+    } else {
+      break;
+    }
+  }
+
+  return systemPrompt ? [systemPrompt, ...res] : res;
+};
+
+/* system 内容截断 */
+export const systemPromptFilter = (prompts: string[], maxTokens: number) => {
+  let splitText = '';
+
+  // 从前往前截取
+  for (let i = 0; i < prompts.length; i++) {
+    const prompt = prompts[i];
+
+    splitText += `${prompt}\n`;
+    const tokens = encode(splitText).length;
+    if (tokens >= maxTokens) {
+      break;
+    }
+  }
+
+  return splitText;
+};

src/types/chat.d.ts

@@ -1,6 +1,7 @@
 export type ChatItemType = {
   obj: 'Human' | 'AI' | 'SYSTEM';
   value: string;
+  deleted?: boolean;
 };
 
 export type ChatSiteItemType = {

src/types/data.d.ts

@@ -1,5 +1,7 @@
 import type { DataSchema } from './mongoSchema';
 
+export type DataType = 'QA' | 'abstract';
+
 export interface DataListItem extends DataSchema {
   trainingData: number;
   totalData: number;

src/types/index.d.ts

@@ -1,8 +1,12 @@
 import type { Mongoose } from 'mongoose';
+import type { RedisClientType } from 'redis';
 
 declare global {
   var mongodb: Mongoose | string | null;
+  var redisClient: RedisClientType | null;
   var generatingQA: boolean;
+  var generatingAbstract: boolean;
+  var generatingVector: boolean;
   var QRCode: any;
   interface Window {
     ['pdfjs-dist/build/pdf']: any;

src/types/model.d.ts

@@ -8,3 +8,12 @@ export interface ModelUpdateParams {
   service: ModelSchema.service;
   security: ModelSchema.security;
 }
+
+export interface ModelDataItemType {
+  id: string;
+  status: 0 | 1; // 1代表向量生成完毕
+  q: string; // 提问词
+  a: string; // 原文
+  modelId: string;
+  userId: string;
+}

src/types/mongoSchema.d.ts

@@ -1,5 +1,6 @@
 import type { ChatItemType } from './chat';
 import { ModelStatusEnum, TrainingStatusEnum, ChatModelNameEnum } from '@/constants/model';
+import type { DataType } from './data';
 
 export type ServiceName = 'openai';
 
@@ -50,6 +51,29 @@ export interface ModelPopulate extends ModelSchema {
   userId: UserModelSchema;
 }
 
+export type ModelDataType = 0 | 1;
+export interface ModelDataSchema {
+  _id: string;
+  modelId: string;
+  userId: string;
+  text: string;
+  q: {
+    id: string;
+    text: string;
+  };
+  status: ModelDataType;
+}
+
+export interface ModelSplitDataSchema {
+  _id: string;
+  userId: string;
+  modelId: string;
+  rawText: string;
+  prompt: string;
+  errorText: string;
+  textList: string[];
+}
+
 export interface TrainingSchema {
   _id: string;
   serviceName: ServiceName;
@@ -68,7 +92,7 @@ export interface ChatSchema {
   modelId: string;
   expiredTime: number;
   loadAmount: number;
-  updateTime: number;
+  updateTime: Date;
   isShare: boolean;
   content: ChatItemType[];
 }
@@ -102,19 +126,21 @@ export interface DataSchema {
   userId: string;
   name: string;
   createTime: string;
+  type: DataType;
 }
 
 export interface DataItemSchema {
   _id: string;
   userId: string;
   dataId: string;
+  type: DataType;
   times: number;
-  temperature: number;
   text: string;
   rawResponse: string[];
   result: {
-    q: string;
+    q?: string;
-    a: string;
+    a?: string;
+    abstract?: string;
   }[];
   status: 0 | 1 | 2;
 }