perf: export data

.env.template

@@ -1,9 +1,6 @@
 # proxy
 # AXIOS_PROXY_HOST=127.0.0.1
 # AXIOS_PROXY_PORT=7890
-# 是否开启队列任务。 1-开启，0-关闭（请求parentUrl去执行任务,单机时直接填1）
-queueTask=1
-parentUrl=https://hostname/api/openapi/startEvents
 # email
 MY_MAIL=xxx@qq.com
 MAILE_CODE=xxx
@@ -21,7 +18,8 @@ SENSITIVE_CHECK=1
 # openai
 # OPENAI_BASE_URL=https://api.openai.com/v1
 # OPENAI_BASE_URL_AUTH=可选的安全凭证(不需要的时候，记得去掉)
-OPENAIKEY=sk-xxx
+OPENAIKEY=sk-xxx # 对话用的key
+OPENAI_TRAINING_KEY=sk-xxx # 训练用的key
 GPT4KEY=sk-xxx
 # claude
 CLAUDE_BASE_URL=calude模型请求地址

docs/deploy/fastgpt/docker-compose.yml

@@ -17,7 +17,7 @@ services:
       - /root/fastgpt/pg/data:/var/lib/postgresql/data
       - /etc/localtime:/etc/localtime:ro
   mongodb:
-    image: mongo:6.0.4
+    image: mongo:5.0.18
     container_name: mongo
     restart: always
     ports:
@@ -35,13 +35,10 @@ services:
     network_mode: host
     restart: always
     container_name: fastgpt
-    environment:
+    environment: # 可选的变量，不需要的话需要去掉
       # proxy（可选）
       - AXIOS_PROXY_HOST=127.0.0.1
       - AXIOS_PROXY_PORT=7890
-      # 是否开启队列任务。 1-开启，0-关闭（请求 parentUrl 去执行任务,单机时直接填1）
-      - queueTask=1
-      - parentUrl=https://hostname/api/openapi/startEvents
       # 发送邮箱验证码配置。用的是QQ邮箱。参考 nodeMail 获取MAILE_CODE，自行百度。
       - MY_MAIL=xxxx@qq.com
       - MAILE_CODE=xxxx
@@ -50,11 +47,17 @@ services:
       - aliAccessKeySecret=xxxx
       - aliSignName=xxxxx
       - aliTemplateCode=SMS_xxxx
+      # google V3 安全校验（可选）
+      - CLIENT_GOOGLE_VER_TOKEN=xxx
+      - SERVICE_GOOGLE_VER_TOKEN=xx
+      # QA和向量生成最大进程数
+      - QA_MAX_PROCESS=10
+      - VECTOR_MAX_PROCESS=10
       # token加密凭证（随便填，作为登录凭证）
       - TOKEN_KEY=xxxx
       # root key, 最高权限，可以内部接口互相调用
       - ROOT_KEY=xxx
-      # 是否进行安全校验(1: 开启，0: 关闭)
+      # 是否进行内容安全校验(1: 开启，0: 关闭)
       - SENSITIVE_CHECK=1
       # 和上方mongo镜像的username,password对应
       - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin
@@ -66,7 +69,8 @@ services:
       - PG_PASSWORD=1234 # POSTGRES_PASSWORD
       - PG_DB_NAME=fastgpt # POSTGRES_DB
       # openai
-      - OPENAIKEY=sk-xxxxx
+      - OPENAIKEY=sk-xxxxx,sk-xxx # 对话用的key，多个key，逗号分开
+      - OPENAI_TRAINING_KEY=sk-xxx,sk-xxxx # 训练用的key
       - GPT4KEY=sk-xxx
       - OPENAI_BASE_URL=https://api.openai.com/v1
       - OPENAI_BASE_URL_AUTH=可选的安全凭证

docs/deploy/fastgpt/pg/init.sql

@@ -1,21 +1,20 @@
 set -e
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
 
-CREATE EXTENSION vector;
+CREATE EXTENSION IF NOT EXISTS vector;
 -- init table
-CREATE TABLE modelData (
+CREATE TABLE IF NOT EXISTS modelData (
     id BIGSERIAL PRIMARY KEY,
-    vector VECTOR(1536),
+    vector VECTOR(1536) NOT NULL,
-    status VARCHAR(50) NOT NULL,
     user_id VARCHAR(50) NOT NULL,
-    model_id VARCHAR(50),
+    kb_id VARCHAR(50) NOT NULL,
-    kb_id VARCHAR(50),
     q TEXT NOT NULL,
     a TEXT NOT NULL
 );
--- create index
+-- 索引设置，按需取
-CREATE INDEX modelData_status_index ON modelData USING HASH (status);
+-- CREATE INDEX IF NOT EXISTS modelData_userId_index ON modelData USING HASH (user_id);
-CREATE INDEX modelData_userId_index ON modelData USING HASH (user_id);
+-- CREATE INDEX IF NOT EXISTS modelData_kbId_index ON modelData USING HASH (kb_id);
-CREATE INDEX modelData_userId_index ON modelData USING HASH (model_id);
+-- CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON modelData (md5(q), md5(a), user_id, kb_id);
-CREATE INDEX modelData_kbId_index ON modelData USING HASH (kb_id);
+-- CREATE INDEX IF NOT EXISTS vector_index ON modelData USING ivfflat (vector vector_cosine_ops) WITH (lists = 1000);
-EOSQL
+-- vector 索引，可以到 pg vector 去配置，根据数据量去配置
+EOSQL

docs/deploy/mac.md

@@ -36,7 +36,6 @@ mongo pg
 AXIOS_PROXY_HOST=127.0.0.1
 AXIOS_PROXY_PORT_FAST=7890
 AXIOS_PROXY_PORT_NORMAL=7890
-queueTask=1
 # email
 MY_MAIL= {Your Mail}
 MAILE_CODE={Yoir Mail code}
@@ -48,7 +47,8 @@ aliTemplateCode=SMS_xxx
 # token
 TOKEN_KEY=sswada
 # openai
-OPENAIKEY={Your openapi key}
+OPENAIKEY=sk-xxx # 对话用的key
+OPENAI_TRAINING_KEY=sk-xxx # 训练用的key
 # db
 MONGODB_URI=mongodb://username:password@0.0.0.0:27017/test?authSource=admin
 PG_HOST=0.0.0.0

docs/dev/README.md

@@ -2,7 +2,7 @@
 
 第一次开发，请先[部署教程](../deploy/docker.md)，需要部署数据库.
 
-## 环境变量配置
+## 环境变量配置 (可能更新不及时，以 docker-compose 里的变量为准)
 
 复制.env.template 文件，生成一个.env.local 环境变量文件夹，修改.env.local 里内容。
 
@@ -10,9 +10,6 @@
 # proxy（可选）
 AXIOS_PROXY_HOST=127.0.0.1
 AXIOS_PROXY_PORT=7890
-# 是否开启队列任务。 1-开启，0-关闭（请求parentUrl去执行任务,单机时直接填1）
-queueTask=1
-parentUrl=https://hostname/api/openapi/startEvents
 # email
 MY_MAIL=xxx@qq.com
 MAILE_CODE=xxx
@@ -30,7 +27,8 @@ SENSITIVE_CHECK=1
 # openai
 # OPENAI_BASE_URL=https://api.openai.com/v1
 # OPENAI_BASE_URL_AUTH=可选的安全凭证(不需要的时候，记得去掉)
-OPENAIKEY=sk-xxx
+OPENAIKEY=sk-xxx # 对话用的key
+OPENAI_TRAINING_KEY=sk-xxx # 训练用的key
 GPT4KEY=sk-xxx
 # claude
 CLAUDE_BASE_URL=calude模型请求地址

package.json

@@ -50,10 +50,10 @@
     "react-hook-form": "^7.43.1",
     "react-markdown": "^8.0.5",
     "react-syntax-highlighter": "^15.5.0",
-    "redis": "^4.6.5",
     "rehype-katex": "^6.0.2",
     "remark-gfm": "^3.0.1",
     "remark-math": "^5.1.1",
+    "request-ip": "^3.3.0",
     "sass": "^1.58.3",
     "sharp": "^0.31.3",
     "tunnel": "^0.0.6",
@@ -73,6 +73,7 @@
     "@types/react": "18.0.28",
     "@types/react-dom": "18.0.11",
     "@types/react-syntax-highlighter": "^15.5.6",
+    "@types/request-ip": "^0.0.37",
     "@types/tunnel": "^0.0.3",
     "eslint": "8.34.0",
     "eslint-config-next": "13.1.6",

pnpm-lock.yaml

@@ -25,6 +25,7 @@ specifiers:
   '@types/react': 18.0.28
   '@types/react-dom': 18.0.11
   '@types/react-syntax-highlighter': ^15.5.6
+  '@types/request-ip': ^0.0.37
   '@types/tunnel': ^0.0.3
   axios: ^1.3.3
   cookie: ^0.5.0
@@ -58,10 +59,10 @@ specifiers:
   react-hook-form: ^7.43.1
   react-markdown: ^8.0.5
   react-syntax-highlighter: ^15.5.0
-  redis: ^4.6.5
   rehype-katex: ^6.0.2
   remark-gfm: ^3.0.1
   remark-math: ^5.1.1
+  request-ip: ^3.3.0
   sass: ^1.58.3
   sharp: ^0.31.3
   tunnel: ^0.0.6
@@ -109,10 +110,10 @@ dependencies:
   react-hook-form: registry.npmmirror.com/react-hook-form/7.43.1_react@18.2.0
   react-markdown: registry.npmmirror.com/react-markdown/8.0.5_pmekkgnqduwlme35zpnqhenc34
   react-syntax-highlighter: registry.npmmirror.com/react-syntax-highlighter/15.5.0_react@18.2.0
-  redis: registry.npmmirror.com/redis/4.6.5
   rehype-katex: registry.npmmirror.com/rehype-katex/6.0.2
   remark-gfm: registry.npmmirror.com/remark-gfm/3.0.1
   remark-math: registry.npmmirror.com/remark-math/5.1.1
+  request-ip: 3.3.0
   sass: registry.npmmirror.com/sass/1.58.3
   sharp: registry.npmmirror.com/sharp/0.31.3
   tunnel: registry.npmmirror.com/tunnel/0.0.6
@@ -132,6 +133,7 @@ devDependencies:
   '@types/react': registry.npmmirror.com/@types/react/18.0.28
   '@types/react-dom': registry.npmmirror.com/@types/react-dom/18.0.11
   '@types/react-syntax-highlighter': registry.npmmirror.com/@types/react-syntax-highlighter/15.5.6
+  '@types/request-ip': 0.0.37
   '@types/tunnel': registry.npmmirror.com/@types/tunnel/0.0.3
   eslint: registry.npmmirror.com/eslint/8.34.0
   eslint-config-next: registry.npmmirror.com/eslint-config-next/13.1.6_7kw3g6rralp5ps6mg3uyzz6azm
@@ -302,6 +304,19 @@ packages:
       '@types/unist': 2.0.6
     dev: false
 
+  /@types/node/14.18.42:
+    resolution: {integrity: sha512-xefu+RBie4xWlK8hwAzGh3npDz/4VhF6icY/shU+zv/1fNn+ZVG7T7CRwe9LId9sAYRPxI+59QBPuKL3WpyGRg==}
+    dev: false
+
+  /@types/node/18.14.0:
+    resolution: {integrity: sha512-5EWrvLmglK+imbCJY0+INViFWUHg1AHel1sq4ZVSfdcNqGy9Edv3UB9IIzzg+xPaUcAgZYcfVs2fBcwDeZzU0A==}
+
+  /@types/request-ip/0.0.37:
+    resolution: {integrity: sha512-uw6/i3rQnpznxD7LtLaeuZytLhKZK6bRoTS6XVJlwxIOoOpEBU7bgKoVXDNtOg4Xl6riUKHa9bjMVrL6ESqYlQ==}
+    dependencies:
+      '@types/node': 18.14.0
+    dev: true
+
   /@types/unist/2.0.6:
     resolution: {integrity: sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==}
     dev: false
@@ -347,6 +362,10 @@ packages:
     engines: {node: '>=0.10.0'}
     dev: false
 
+  /request-ip/3.3.0:
+    resolution: {integrity: sha512-cA6Xh6e0fDBBBwH77SLJaJPBmD3nWVAcF9/XAcsrIHdjhFzFiB5aNQFytdjCGPezU3ROwrR11IddKAM08vohxA==}
+    dev: false
+
   /saslprep/1.0.3:
     resolution: {integrity: sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==}
     engines: {node: '>=6'}
@@ -4897,72 +4916,6 @@ packages:
     version: 2.11.6
     dev: false
 
-  registry.npmmirror.com/@redis/bloom/1.2.0_@redis+client@1.5.6:
-    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/bloom/-/bloom-1.2.0.tgz}
-    id: registry.npmmirror.com/@redis/bloom/1.2.0
-    name: '@redis/bloom'
-    version: 1.2.0
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
-  registry.npmmirror.com/@redis/client/1.5.6:
-    resolution: {integrity: sha512-dFD1S6je+A47Lj22jN/upVU2fj4huR7S9APd7/ziUXsIXDL+11GPYti4Suv5y8FuXaN+0ZG4JF+y1houEJ7ToA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/client/-/client-1.5.6.tgz}
-    name: '@redis/client'
-    version: 1.5.6
-    engines: {node: '>=14'}
-    dependencies:
-      cluster-key-slot: registry.npmmirror.com/cluster-key-slot/1.1.2
-      generic-pool: registry.npmmirror.com/generic-pool/3.9.0
-      yallist: registry.npmmirror.com/yallist/4.0.0
-    dev: false
-
-  registry.npmmirror.com/@redis/graph/1.1.0_@redis+client@1.5.6:
-    resolution: {integrity: sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/graph/-/graph-1.1.0.tgz}
-    id: registry.npmmirror.com/@redis/graph/1.1.0
-    name: '@redis/graph'
-    version: 1.1.0
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
-  registry.npmmirror.com/@redis/json/1.0.4_@redis+client@1.5.6:
-    resolution: {integrity: sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/json/-/json-1.0.4.tgz}
-    id: registry.npmmirror.com/@redis/json/1.0.4
-    name: '@redis/json'
-    version: 1.0.4
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
-  registry.npmmirror.com/@redis/search/1.1.2_@redis+client@1.5.6:
-    resolution: {integrity: sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/search/-/search-1.1.2.tgz}
-    id: registry.npmmirror.com/@redis/search/1.1.2
-    name: '@redis/search'
-    version: 1.1.2
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
-  registry.npmmirror.com/@redis/time-series/1.0.4_@redis+client@1.5.6:
-    resolution: {integrity: sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/time-series/-/time-series-1.0.4.tgz}
-    id: registry.npmmirror.com/@redis/time-series/1.0.4
-    name: '@redis/time-series'
-    version: 1.0.4
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
   registry.npmmirror.com/@rushstack/eslint-patch/1.2.0:
     resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@rushstack/eslint-patch/-/eslint-patch-1.2.0.tgz}
     name: '@rushstack/eslint-patch'
@@ -5287,16 +5240,11 @@ packages:
     version: 12.20.55
     dev: false
 
-  registry.npmmirror.com/@types/node/14.18.42:
-    resolution: {integrity: sha512-xefu+RBie4xWlK8hwAzGh3npDz/4VhF6icY/shU+zv/1fNn+ZVG7T7CRwe9LId9sAYRPxI+59QBPuKL3WpyGRg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node/-/node-14.18.42.tgz}
-    name: '@types/node'
-    version: 14.18.42
-    dev: false
-
   registry.npmmirror.com/@types/node/18.14.0:
     resolution: {integrity: sha512-5EWrvLmglK+imbCJY0+INViFWUHg1AHel1sq4ZVSfdcNqGy9Edv3UB9IIzzg+xPaUcAgZYcfVs2fBcwDeZzU0A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node/-/node-18.14.0.tgz}
     name: '@types/node'
     version: 18.14.0
+    dev: true
 
   registry.npmmirror.com/@types/nodemailer/6.4.7:
     resolution: {integrity: sha512-f5qCBGAn/f0qtRcd4SEn88c8Fp3Swct1731X4ryPKqS61/A3LmmzN8zaEz7hneJvpjFbUUgY7lru/B/7ODTazg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/nodemailer/-/nodemailer-6.4.7.tgz}
@@ -5395,7 +5343,7 @@ packages:
     name: '@types/whatwg-url'
     version: 8.2.2
     dependencies:
-      '@types/node': registry.npmmirror.com/@types/node/18.14.0
+      '@types/node': 18.14.0
       '@types/webidl-conversions': registry.npmmirror.com/@types/webidl-conversions/7.0.0
     dev: false
 
@@ -5404,7 +5352,7 @@ packages:
     name: '@types/xml2js'
     version: 0.4.11
     dependencies:
-      '@types/node': registry.npmmirror.com/@types/node/18.14.0
+      '@types/node': 18.14.0
     dev: false
 
   registry.npmmirror.com/@typescript-eslint/parser/5.52.0_7kw3g6rralp5ps6mg3uyzz6azm:
@@ -6117,13 +6065,6 @@ packages:
     version: 0.0.1
     dev: false
 
-  registry.npmmirror.com/cluster-key-slot/1.1.2:
-    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz}
-    name: cluster-key-slot
-    version: 1.1.2
-    engines: {node: '>=0.10.0'}
-    dev: false
-
   registry.npmmirror.com/color-convert/1.9.3:
     resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/color-convert/-/color-convert-1.9.3.tgz}
     name: color-convert
@@ -7514,13 +7455,6 @@ packages:
     version: 1.2.3
     dev: true
 
-  registry.npmmirror.com/generic-pool/3.9.0:
-    resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/generic-pool/-/generic-pool-3.9.0.tgz}
-    name: generic-pool
-    version: 3.9.0
-    engines: {node: '>= 4'}
-    dev: false
-
   registry.npmmirror.com/gensync/1.0.0-beta.2:
     resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/gensync/-/gensync-1.0.0-beta.2.tgz}
     name: gensync
@@ -7926,7 +7860,7 @@ packages:
     name: httpx
     version: 2.2.7
     dependencies:
-      '@types/node': registry.npmmirror.com/@types/node/14.18.42
+      '@types/node': 14.18.42
       debug: registry.npmmirror.com/debug/4.3.4
     transitivePeerDependencies:
       - supports-color
@@ -10515,19 +10449,6 @@ packages:
       picomatch: registry.npmmirror.com/picomatch/2.3.1
     dev: false
 
-  registry.npmmirror.com/redis/4.6.5:
-    resolution: {integrity: sha512-O0OWA36gDQbswOdUuAhRL6mTZpHFN525HlgZgDaVNgCJIAZR3ya06NTESb0R+TUZ+BFaDpz6NnnVvoMx9meUFg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/redis/-/redis-4.6.5.tgz}
-    name: redis
-    version: 4.6.5
-    dependencies:
-      '@redis/bloom': registry.npmmirror.com/@redis/bloom/1.2.0_@redis+client@1.5.6
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-      '@redis/graph': registry.npmmirror.com/@redis/graph/1.1.0_@redis+client@1.5.6
-      '@redis/json': registry.npmmirror.com/@redis/json/1.0.4_@redis+client@1.5.6
-      '@redis/search': registry.npmmirror.com/@redis/search/1.1.2_@redis+client@1.5.6
-      '@redis/time-series': registry.npmmirror.com/@redis/time-series/1.0.4_@redis+client@1.5.6
-    dev: false
-
   registry.npmmirror.com/reflect-metadata/0.1.13:
     resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz}
     name: reflect-metadata

src/api/plugins/kb.ts

@@ -1,8 +1,11 @@
 import { GET, POST, PUT, DELETE } from '../request';
 import type { KbItemType } from '@/types/plugin';
 import { RequestPaging } from '@/types/index';
-import { SplitTextTypEnum } from '@/constants/plugin';
+import { TrainingModeEnum } from '@/constants/plugin';
-import { KbDataItemType } from '@/types/plugin';
+import {
+  Props as PushDataProps,
+  Response as PushDateResponse
+} from '@/pages/api/openapi/kb/pushData';
 
 export type KbUpdateParams = { id: string; name: string; tags: string; avatar: string };
 
@@ -29,16 +32,22 @@ export const getKbDataList = (data: GetKbDataListProps) =>
  * 获取导出数据（不分页）
  */
 export const getExportDataList = (kbId: string) =>
-  GET<[string, string][]>(`/plugins/kb/data/exportModelData?kbId=${kbId}`);
+  GET<[string, string][]>(
+    `/plugins/kb/data/exportModelData`,
+    { kbId },
+    {
+      timeout: 600000
+    }
+  );
 
 /**
  * 获取模型正在拆分数据的数量
  */
-export const getTrainingData = (kbId: string) =>
+export const getTrainingData = (data: { kbId: string; init: boolean }) =>
-  GET<{
+  POST<{
-    splitDataQueue: number;
+    qaListLen: number;
-    embeddingQueue: number;
+    vectorListLen: number;
-  }>(`/plugins/kb/data/getTrainingData?kbId=${kbId}`);
+  }>(`/plugins/kb/data/getTrainingData`, data);
 
 export const getKbDataItemById = (dataId: string) =>
   GET(`/plugins/kb/data/getDataById`, { dataId });
@@ -46,10 +55,8 @@ export const getKbDataItemById = (dataId: string) =>
 /**
  * 直接push数据
  */
-export const postKbDataFromList = (data: {
+export const postKbDataFromList = (data: PushDataProps) =>
-  kbId: string;
+  POST<PushDateResponse>(`/openapi/kb/pushData`, data);
-  data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
-}) => POST(`/openapi/kb/pushData`, data);
 
 /**
  * 更新一条数据
@@ -69,5 +76,5 @@ export const postSplitData = (data: {
   kbId: string;
   chunks: string[];
   prompt: string;
-  mode: `${SplitTextTypEnum}`;
+  mode: `${TrainingModeEnum}`;
-}) => POST(`/openapi/text/splitText`, data);
+}) => POST(`/openapi/text/pushData`, data);

src/api/request.ts

@@ -5,6 +5,7 @@ import { TOKEN_ERROR_CODE } from '@/service/errorCode';
 interface ConfigType {
   headers?: { [key: string]: string };
   hold?: boolean;
+  timeout?: number;
 }
 interface ResponseDataType {
   code: number;

src/api/system.ts

@@ -1,6 +1,7 @@
 import { GET, POST, PUT } from './request';
 import type { ChatModelItemType } from '@/constants/model';
+import type { InitDateResponse } from '@/pages/api/system/getInitData';
 
-export const getFilling = () => GET<{ beianText: string }>('/system/getFiling');
+export const getInitData = () => GET<InitDateResponse>('/system/getInitData');
 
 export const getSystemModelList = () => GET<ChatModelItemType[]>('/system/getModels');

src/api/user.ts

@@ -6,13 +6,11 @@ import { UserBillType, UserType, UserUpdateParams } from '@/types/user';
 import type { PagingData, RequestPaging } from '@/types';
 import { PaySchema } from '@/types/mongoSchema';
 
-export const sendAuthCode = ({
+export const sendAuthCode = (data: {
-  username,
-  type
-}: {
   username: string;
   type: `${UserAuthTypeEnum}`;
-}) => GET('/user/sendAuthCode', { username, type });
+  googleToken: string;
+}) => POST('/user/sendAuthCode', data);
 
 export const getTokenLogin = () => GET<UserType>('/user/tokenLogin');
 

src/components/Avatar/index.tsx

@@ -7,7 +7,7 @@ const Avatar = ({ w = '30px', ...props }: ImageProps) => {
     <Image
       fallbackSrc="/icon/logo.png"
       borderRadius={'50%'}
-      objectFit={'contain'}
+      objectFit={'cover'}
       alt=""
       w={w}
       h={w}

src/constants/model.ts

@@ -30,7 +30,7 @@ export const ChatModelMap = {
     chatModel: OpenAiChatEnum.GPT35,
     name: 'ChatGpt',
     contextMaxToken: 4096,
-    systemMaxToken: 2400,
+    systemMaxToken: 2700,
     maxTemperature: 1.2,
     price: 2.5
   },
@@ -38,7 +38,7 @@ export const ChatModelMap = {
     chatModel: OpenAiChatEnum.GPT4,
     name: 'Gpt4',
     contextMaxToken: 8000,
-    systemMaxToken: 3000,
+    systemMaxToken: 4000,
     maxTemperature: 1.2,
     price: 50
   },
@@ -46,7 +46,7 @@ export const ChatModelMap = {
     chatModel: OpenAiChatEnum.GPT432k,
     name: 'Gpt4-32k',
     contextMaxToken: 32000,
-    systemMaxToken: 3000,
+    systemMaxToken: 8000,
     maxTemperature: 1.2,
     price: 90
   },
@@ -54,7 +54,7 @@ export const ChatModelMap = {
     chatModel: ClaudeEnum.Claude,
     name: 'Claude(免费体验)',
     contextMaxToken: 9000,
-    systemMaxToken: 2400,
+    systemMaxToken: 2700,
     maxTemperature: 1,
     price: 0
   }
@@ -96,39 +96,29 @@ export const formatModelStatus = {
   }
 };
 
-export enum ModelDataStatusEnum {
-  ready = 'ready',
-  waiting = 'waiting'
-}
-
-export const ModelDataStatusMap: Record<`${ModelDataStatusEnum}`, string> = {
-  ready: '训练完成',
-  waiting: '训练中'
-};
-
 /* 知识库搜索时的配置 */
 // 搜索方式
-export enum ModelVectorSearchModeEnum {
+export enum appVectorSearchModeEnum {
   hightSimilarity = 'hightSimilarity', // 高相似度+禁止回复
   lowSimilarity = 'lowSimilarity', // 低相似度
   noContext = 'noContex' // 高相似度+无上下文回复
 }
 export const ModelVectorSearchModeMap: Record<
-  `${ModelVectorSearchModeEnum}`,
+  `${appVectorSearchModeEnum}`,
   {
     text: string;
     similarity: number;
   }
 > = {
-  [ModelVectorSearchModeEnum.hightSimilarity]: {
+  [appVectorSearchModeEnum.hightSimilarity]: {
     text: '高相似度, 无匹配时拒绝回复',
     similarity: 0.18
   },
-  [ModelVectorSearchModeEnum.noContext]: {
+  [appVectorSearchModeEnum.noContext]: {
     text: '高相似度，无匹配时直接回复',
     similarity: 0.18
   },
-  [ModelVectorSearchModeEnum.lowSimilarity]: {
+  [appVectorSearchModeEnum.lowSimilarity]: {
     text: '低相似度匹配',
     similarity: 0.7
   }
@@ -143,7 +133,7 @@ export const defaultModel: ModelSchema = {
   updateTime: Date.now(),
   chat: {
     relatedKbs: [],
-    searchMode: ModelVectorSearchModeEnum.hightSimilarity,
+    searchMode: appVectorSearchModeEnum.hightSimilarity,
     systemPrompt: '',
     temperature: 0,
     chatModel: OpenAiChatEnum.GPT35

src/constants/plugin.ts

@@ -1,14 +1,8 @@
-export enum SplitTextTypEnum {
+export enum TrainingModeEnum {
   'qa' = 'qa',
-  'subsection' = 'subsection'
+  'index' = 'index'
-}
-
-export enum PluginTypeEnum {
-  LLM = 'LLM',
-  Text = 'Text',
-  Function = 'Function'
-}
-
-export enum PluginParamsTypeEnum {
-  'Text' = 'text'
 }
+export const TrainingTypeMap = {
+  [TrainingModeEnum.qa]: 'qa',
+  [TrainingModeEnum.index]: 'index'
+};

src/hooks/useSendCode.ts

@@ -3,8 +3,13 @@ import { sendAuthCode } from '@/api/user';
 import { UserAuthTypeEnum } from '@/constants/common';
 let timer: any;
 import { useToast } from './useToast';
+import { getClientToken } from '@/utils/plugin/google';
+import { useGlobalStore } from '@/store/global';
 
 export const useSendCode = () => {
+  const {
+    initData: { googleVerKey }
+  } = useGlobalStore();
   const { toast } = useToast();
   const [codeSending, setCodeSending] = useState(false);
   const [codeCountDown, setCodeCountDown] = useState(0);
@@ -24,7 +29,8 @@ export const useSendCode = () => {
       try {
         await sendAuthCode({
           username,
-          type
+          type,
+          googleToken: await getClientToken(googleVerKey)
         });
         setCodeCountDown(60);
         timer = setInterval(() => {
@@ -48,7 +54,7 @@ export const useSendCode = () => {
       }
       setCodeSending(false);
     },
-    [toast]
+    [googleVerKey, toast]
   );
 
   return {

src/pages/_app.tsx

@@ -10,7 +10,7 @@ import NProgress from 'nprogress'; //nprogress module
 import Router from 'next/router';
 import 'nprogress/nprogress.css';
 import '../styles/reset.scss';
-import { useToast } from '@/hooks/useToast';
+import { useGlobalStore } from '@/store/global';
 
 //Binding events.
 Router.events.on('routeChangeStart', () => NProgress.start());
@@ -29,23 +29,20 @@ const queryClient = new QueryClient({
 });
 
 export default function App({ Component, pageProps }: AppProps) {
-  const { toast } = useToast();
+  const {
-  // 校验是否支持 click 事件
+    loadInitData,
+    initData: { googleVerKey }
+  } = useGlobalStore();
+
   useEffect(() => {
-    if (typeof document.createElement('div').click !== 'function') {
+    loadInitData();
-      toast({
+  }, []);
-        title: '你的浏览器版本过低',
-        status: 'warning'
-      });
-    }
-  }, [toast]);
 
   return (
     <>
       <Head>
         <title>Fast GPT</title>
         <meta name="description" content="Generated by Fast GPT" />
-
         <meta
           name="viewport"
           content="width=device-width,initial-scale=1.0,maximum-scale=1.0,minimum-scale=1.0,user-scalable=no, viewport-fit=cover"
@@ -55,6 +52,12 @@ export default function App({ Component, pageProps }: AppProps) {
       <Script src="/js/qrcode.min.js" strategy="lazyOnload"></Script>
       <Script src="/js/pdf.js" strategy="lazyOnload"></Script>
       <Script src="/js/html2pdf.bundle.min.js" strategy="lazyOnload"></Script>
+      {googleVerKey && (
+        <Script
+          src={`https://www.recaptcha.net/recaptcha/api.js?render=${googleVerKey}`}
+          strategy="lazyOnload"
+        ></Script>
+      )}
       <Script src="/js/particles.js"></Script>
       <QueryClientProvider client={queryClient}>
         <ChakraProvider theme={theme}>

src/pages/api/admin/countTraining.ts

@@ -0,0 +1,37 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { authUser } from '@/service/utils/auth';
+import { connectToDatabase, TrainingData } from '@/service/mongo';
+import { TrainingModeEnum } from '@/constants/plugin';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    await authUser({ req, authRoot: true });
+
+    await connectToDatabase();
+
+    // split queue data
+    const result = await TrainingData.aggregate([
+      {
+        $group: {
+          _id: '$mode',
+          count: { $sum: 1 }
+        }
+      }
+    ]);
+
+    jsonRes(res, {
+      data: {
+        qaListLen: result.find((item) => item._id === TrainingModeEnum.qa)?.count || 0,
+        vectorListLen: result.find((item) => item._id === TrainingModeEnum.index)?.count || 0
+      }
+    });
+  } catch (error) {
+    console.log(error);
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}

src/pages/api/chat/saveChat.ts

@@ -58,7 +58,11 @@ export async function saveChat({
     obj: item.obj,
     value: item.value,
     systemPrompt: item.systemPrompt,
-    quote: item.quote || []
+    quote:
+      item.quote?.map((item) => ({
+        ...item,
+        isEdit: false
+      })) || []
   }));
 
   // 没有 chatId, 创建一个对话

src/pages/api/openapi/kb/appKbSearch.ts

@@ -5,12 +5,11 @@ import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
 import type { ChatItemSimpleType } from '@/types/chat';
 import type { ModelSchema } from '@/types/mongoSchema';
-import { ModelVectorSearchModeEnum } from '@/constants/model';
+import { appVectorSearchModeEnum } from '@/constants/model';
 import { authModel } from '@/service/utils/auth';
 import { ChatModelMap } from '@/constants/model';
 import { ChatRoleEnum } from '@/constants/chat';
 import { openaiEmbedding } from '../plugin/openaiEmbedding';
-import { ModelDataStatusEnum } from '@/constants/model';
 import { modelToolMap } from '@/utils/plugin';
 
 export type QuoteItemType = { id: string; q: string; a: string; isEdit: boolean };
@@ -92,7 +91,8 @@ export async function appKbSearch({
   // get vector
   const promptVectors = await openaiEmbedding({
     userId,
-    input
+    input,
+    type: 'chat'
   });
 
   // search kb
@@ -101,8 +101,6 @@ export async function appKbSearch({
       PgClient.select<QuoteItemType>('modelData', {
         fields: ['id', 'q', 'a'],
         where: [
-          ['status', ModelDataStatusEnum.ready],
-          'AND',
           `kb_id IN (${model.chat.relatedKbs.map((item) => `'${item}'`).join(',')})`,
           'AND',
           `vector <=> '[${promptVector}]' < ${similarity}`
@@ -138,7 +136,7 @@ export async function appKbSearch({
         obj: ChatRoleEnum.System,
         value: model.chat.systemPrompt
       }
-    : model.chat.searchMode === ModelVectorSearchModeEnum.noContext
+    : model.chat.searchMode === appVectorSearchModeEnum.noContext
     ? {
         obj: ChatRoleEnum.System,
         value: `知识库是关于"${model.name}"的内容,根据知识库内容回答问题.`
@@ -176,7 +174,7 @@ export async function appKbSearch({
   const systemPrompt = sliceResult.flat().join('\n').trim();
 
   /* 高相似度+不回复 */
-  if (!systemPrompt && model.chat.searchMode === ModelVectorSearchModeEnum.hightSimilarity) {
+  if (!systemPrompt && model.chat.searchMode === appVectorSearchModeEnum.hightSimilarity) {
     return {
       code: 201,
       rawSearch: [],
@@ -190,7 +188,7 @@ export async function appKbSearch({
     };
   }
   /* 高相似度+无上下文，不添加额外知识,仅用系统提示词 */
-  if (!systemPrompt && model.chat.searchMode === ModelVectorSearchModeEnum.noContext) {
+  if (!systemPrompt && model.chat.searchMode === appVectorSearchModeEnum.noContext) {
     return {
       code: 200,
       rawSearch: [],

src/pages/api/openapi/kb/pushData.ts

@@ -1,84 +1,45 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import type { KbDataItemType } from '@/types/plugin';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase } from '@/service/mongo';
+import { connectToDatabase, TrainingData } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
-import { generateVector } from '@/service/events/generateVector';
-import { PgClient, insertKbItem } from '@/service/pg';
 import { authKb } from '@/service/utils/auth';
 import { withNextCors } from '@/service/utils/tools';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { startQueue } from '@/service/utils/tools';
+import { PgClient } from '@/service/pg';
+
+export type Props = {
+  kbId: string;
+  data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
+  mode: `${TrainingModeEnum}`;
+  prompt?: string;
+};
+
+export type Response = {
+  insertLen: number;
+};
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    const {
+    const { kbId, data, mode, prompt } = req.body as Props;
-      kbId,
-      data,
-      formatLineBreak = true
-    } = req.body as {
-      kbId: string;
-      formatLineBreak?: boolean;
-      data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
-    };
 
     if (!kbId || !Array.isArray(data)) {
       throw new Error('缺少参数');
     }
-
     await connectToDatabase();
 
     // 凭证校验
     const { userId } = await authUser({ req });
 
-    await authKb({
+    jsonRes<Response>(res, {
-      userId,
+      data: await pushDataToKb({
-      kbId
+        kbId,
-    });
+        data,
-
+        userId,
-    // 过滤重复的内容
+        mode,
-    const searchRes = await Promise.allSettled(
+        prompt
-      data.map(async ({ q, a = '' }) => {
-        if (!q) {
-          return Promise.reject('q为空');
-        }
-
-        if (formatLineBreak) {
-          q = q.replace(/\\n/g, '\n');
-          a = a.replace(/\\n/g, '\n');
-        }
-
-        // Exactly the same data, not push
-        try {
-          const count = await PgClient.count('modelData', {
-            where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['q', q], 'AND', ['a', a]]
-          });
-          if (count > 0) {
-            return Promise.reject('已经存在');
-          }
-        } catch (error) {
-          error;
-        }
-        return Promise.resolve({
-          q,
-          a
-        });
       })
-    );
-    const filterData = searchRes
-      .filter((item) => item.status === 'fulfilled')
-      .map<{ q: string; a: string }>((item: any) => item.value);
-
-    // 插入记录
-    const insertRes = await insertKbItem({
-      userId,
-      kbId,
-      data: filterData
-    });
-
-    generateVector();
-
-    jsonRes(res, {
-      message: `共插入 ${insertRes.rowCount} 条数据`,
-      data: insertRes.rowCount
     });
   } catch (err) {
     jsonRes(res, {
@@ -88,10 +49,100 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
   }
 });
 
+export async function pushDataToKb({
+  userId,
+  kbId,
+  data,
+  mode,
+  prompt
+}: { userId: string } & Props): Promise<Response> {
+  await authKb({
+    userId,
+    kbId
+  });
+
+  // 过滤重复的 qa 内容
+  const set = new Set();
+  const filterData: {
+    a: string;
+    q: string;
+  }[] = [];
+
+  data.forEach((item) => {
+    const text = item.q + item.a;
+    if (!set.has(text)) {
+      filterData.push(item);
+      set.add(text);
+    }
+  });
+
+  // 数据库去重
+  const insertData = (
+    await Promise.allSettled(
+      filterData.map(async ({ q, a = '' }) => {
+        if (mode !== TrainingModeEnum.index) {
+          return Promise.resolve({
+            q,
+            a
+          });
+        }
+
+        if (!q) {
+          return Promise.reject('q为空');
+        }
+
+        q = q.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
+        a = a.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
+
+        // Exactly the same data, not push
+        try {
+          const { rows } = await PgClient.query(`
+            SELECT COUNT(*) > 0 AS exists
+            FROM  modelData 
+            WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}'
+          `);
+          const exists = rows[0]?.exists || false;
+
+          if (exists) {
+            return Promise.reject('已经存在');
+          }
+        } catch (error) {
+          console.log(error);
+          error;
+        }
+        return Promise.resolve({
+          q,
+          a
+        });
+      })
+    )
+  )
+    .filter((item) => item.status === 'fulfilled')
+    .map<{ q: string; a: string }>((item: any) => item.value);
+
+  // 插入记录
+  await TrainingData.insertMany(
+    insertData.map((item) => ({
+      q: item.q,
+      a: item.a,
+      userId,
+      kbId,
+      mode,
+      prompt
+    }))
+  );
+
+  insertData.length > 0 && startQueue();
+
+  return {
+    insertLen: insertData.length
+  };
+}
+
 export const config = {
   api: {
     bodyParser: {
-      sizeLimit: '100mb'
+      sizeLimit: '20mb'
     }
   }
 };

src/pages/api/openapi/kb/updateData.ts

@@ -1,14 +1,13 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
-import { ModelDataStatusEnum } from '@/constants/model';
-import { generateVector } from '@/service/events/generateVector';
 import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
+import { openaiEmbedding } from '../plugin/openaiEmbedding';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    const { dataId, a, q } = req.body as { dataId: string; a: string; q?: string };
+    const { dataId, a = '', q = '' } = req.body as { dataId: string; a?: string; q?: string };
 
     if (!dataId) {
       throw new Error('缺少参数');
@@ -17,22 +16,32 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     // 凭证校验
     const { userId } = await authUser({ req });
 
+    // get vector
+    const vector = await (async () => {
+      if (q) {
+        return openaiEmbedding({
+          userId,
+          input: [q],
+          type: 'chat'
+        });
+      }
+      return [];
+    })();
+
     // 更新 pg 内容.仅修改a，不需要更新向量。
     await PgClient.update('modelData', {
       where: [['id', dataId], 'AND', ['user_id', userId]],
       values: [
-        { key: 'a', value: a },
+        { key: 'a', value: a.replace(/'/g, '"') },
         ...(q
           ? [
-              { key: 'q', value: q },
+              { key: 'q', value: q.replace(/'/g, '"') },
-              { key: 'status', value: ModelDataStatusEnum.waiting }
+              { key: 'vector', value: `[${vector[0]}]` }
             ]
           : [])
       ]
     });
 
-    q && generateVector();
-
     jsonRes(res);
   } catch (err) {
     jsonRes(res, {

src/pages/api/openapi/plugin/openaiEmbedding.ts

@@ -1,30 +1,31 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
-import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
 import { getApiKey } from '@/service/utils/auth';
 import { getOpenAIApi } from '@/service/utils/chat/openai';
 import { embeddingModel } from '@/constants/model';
 import { axiosConfig } from '@/service/utils/tools';
 import { pushGenerateVectorBill } from '@/service/events/pushBill';
+import { ApiKeyType } from '@/service/utils/auth';
 
 type Props = {
   input: string[];
+  type?: ApiKeyType;
 };
 type Response = number[][];
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     const { userId } = await authUser({ req });
-    let { input } = req.query as Props;
+    let { input, type } = req.query as Props;
 
     if (!Array.isArray(input)) {
       throw new Error('缺少参数');
     }
 
     jsonRes<Response>(res, {
-      data: await openaiEmbedding({ userId, input, mustPay: true })
+      data: await openaiEmbedding({ userId, input, mustPay: true, type })
     });
   } catch (err) {
     console.log(err);
@@ -38,12 +39,14 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
 export async function openaiEmbedding({
   userId,
   input,
-  mustPay = false
+  mustPay = false,
+  type = 'chat'
 }: { userId: string; mustPay?: boolean } & Props) {
   const { userOpenAiKey, systemAuthKey } = await getApiKey({
     model: 'gpt-3.5-turbo',
     userId,
-    mustPay
+    mustPay,
+    type
   });
 
   // 获取 chatAPI

src/pages/api/openapi/startEvents.ts

@@ -1,19 +0,0 @@
-// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { generateQA } from '@/service/events/generateQA';
-import { generateVector } from '@/service/events/generateVector';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    generateQA();
-    generateVector();
-
-    jsonRes(res);
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}

src/pages/api/openapi/text/gptMessagesSlice.ts

@@ -62,5 +62,5 @@ export function gpt_chatItemTokenSlice({
     }
   }
 
-  return result;
+  return result.length === 0 && messages[0] ? [messages[0]] : result;
 }

src/pages/api/openapi/text/sensitiveCheck.ts

@@ -17,7 +17,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const { input } = req.body as TextPluginRequestParams;
 
     const response = await axios({
-      ...axiosConfig(getSystemOpenAiKey()),
+      ...axiosConfig(getSystemOpenAiKey('chat')),
       method: 'POST',
       url: `/moderations`,
       data: {

src/pages/api/openapi/text/splitText.ts

@@ -1,73 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, SplitData } from '@/service/mongo';
-import { authKb, authUser } from '@/service/utils/auth';
-import { generateVector } from '@/service/events/generateVector';
-import { generateQA } from '@/service/events/generateQA';
-import { insertKbItem } from '@/service/pg';
-import { SplitTextTypEnum } from '@/constants/plugin';
-import { withNextCors } from '@/service/utils/tools';
-
-/* split text */
-export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    const { chunks, kbId, prompt, mode } = req.body as {
-      kbId: string;
-      chunks: string[];
-      prompt: string;
-      mode: `${SplitTextTypEnum}`;
-    };
-    if (!chunks || !kbId || !prompt) {
-      throw new Error('参数错误');
-    }
-    await connectToDatabase();
-
-    const { userId } = await authUser({ req });
-
-    // 验证是否是该用户的 model
-    await authKb({
-      kbId,
-      userId
-    });
-
-    if (mode === SplitTextTypEnum.qa) {
-      // 批量QA拆分插入数据
-      await SplitData.create({
-        userId,
-        kbId,
-        textList: chunks,
-        prompt
-      });
-
-      generateQA();
-    } else if (mode === SplitTextTypEnum.subsection) {
-      // 待优化，直接调用另一个接口
-      // 插入记录
-      await insertKbItem({
-        userId,
-        kbId,
-        data: chunks.map((item) => ({
-          q: item,
-          a: ''
-        }))
-      });
-
-      generateVector();
-    }
-
-    jsonRes(res);
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-});
-
-export const config = {
-  api: {
-    bodyParser: {
-      sizeLimit: '100mb'
-    }
-  }
-};

src/pages/api/plugins/kb/data/exportModelData.ts

@@ -69,3 +69,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
   }
 }
+
+export const config = {
+  api: {
+    bodyParser: {
+      sizeLimit: '100mb'
+    }
+  }
+};

src/pages/api/plugins/kb/data/getDataById.ts

@@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     const where: any = [['user_id', userId], 'AND', ['id', dataId]];
 
     const searchRes = await PgClient.select<PgKBDataItemType>('modelData', {
-      fields: ['id', 'q', 'a', 'status'],
+      fields: ['id', 'q', 'a'],
       where,
       limit: 1
     });

src/pages/api/plugins/kb/data/getDataList.ts

@@ -35,7 +35,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     ];
 
     const searchRes = await PgClient.select<PgKBDataItemType>('modelData', {
-      fields: ['id', 'q', 'a', 'status'],
+      fields: ['id', 'q', 'a'],
       where,
       order: [{ field: 'id', mode: 'DESC' }],
       limit: pageSize,

src/pages/api/plugins/kb/data/getTrainingData.ts

@@ -1,14 +1,15 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, SplitData, Model } from '@/service/mongo';
+import { connectToDatabase, TrainingData } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
-import { ModelDataStatusEnum } from '@/constants/model';
+import { TrainingModeEnum } from '@/constants/plugin';
-import { PgClient } from '@/service/pg';
+import { Types } from 'mongoose';
+import { startQueue } from '@/service/utils/tools';
 
 /* 拆分数据成QA */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    const { kbId } = req.query as { kbId: string };
+    const { kbId, init = false } = req.body as { kbId: string; init: boolean };
     if (!kbId) {
       throw new Error('参数错误');
     }
@@ -17,29 +18,31 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     const { userId } = await authUser({ req, authToken: true });
 
     // split queue data
-    const data = await SplitData.find({
+    const result = await TrainingData.aggregate([
-      userId,
+      {
-      kbId,
+        $match: {
-      textList: { $exists: true, $not: { $size: 0 } }
+          userId: new Types.ObjectId(userId),
-    });
+          kbId: new Types.ObjectId(kbId)
-
+        }
-    // embedding queue data
+      },
-    const embeddingData = await PgClient.count('modelData', {
+      {
-      where: [
+        $group: {
-        ['user_id', userId],
+          _id: '$mode',
-        'AND',
+          count: { $sum: 1 }
-        ['kb_id', kbId],
+        }
-        'AND',
+      }
-        ['status', ModelDataStatusEnum.waiting]
+    ]);
-      ]
-    });
 
     jsonRes(res, {
       data: {
-        splitDataQueue: data.map((item) => item.textList).flat().length,
+        qaListLen: result.find((item) => item._id === TrainingModeEnum.qa)?.count || 0,
-        embeddingQueue: embeddingData
+        vectorListLen: result.find((item) => item._id === TrainingModeEnum.index)?.count || 0
       }
     });
+
+    if (init) {
+      startQueue();
+    }
   } catch (err) {
     jsonRes(res, {
       code: 500,

src/pages/api/plugins/kb/delete.ts

@@ -1,8 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, KB } from '@/service/mongo';
+import { connectToDatabase, KB, Model, TrainingData } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { PgClient } from '@/service/pg';
+import { Types } from 'mongoose';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -19,19 +20,30 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
     await connectToDatabase();
 
-    // delete mongo data
-    await KB.findOneAndDelete({
-      _id: id,
-      userId
-    });
-
     // delete all pg data
-    // 删除 pg 中所有该模型的数据
     await PgClient.delete('modelData', {
       where: [['user_id', userId], 'AND', ['kb_id', id]]
     });
 
+    // delete training data
+    await TrainingData.deleteMany({
+      userId,
+      kbId: id
+    });
+
     // delete related model
+    await Model.updateMany(
+      {
+        userId
+      },
+      { $pull: { 'chat.relatedKbs': new Types.ObjectId(id) } }
+    );
+
+    // delete kb data
+    await KB.findOneAndDelete({
+      _id: id,
+      userId
+    });
 
     jsonRes(res);
   } catch (err) {

src/pages/api/system/getInitData.ts

@@ -2,10 +2,16 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 
+export type InitDateResponse = {
+  beianText: string;
+  googleVerKey: string;
+};
+
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  jsonRes(res, {
+  jsonRes<InitDateResponse>(res, {
     data: {
-      beianText: process.env.SAFE_BEIAN_TEXT || ''
+      beianText: process.env.SAFE_BEIAN_TEXT || '',
+      googleVerKey: process.env.CLIENT_GOOGLE_VER_TOKEN || ''
     }
   });
 }

src/pages/api/user/sendAuthCode.ts

@@ -7,15 +7,29 @@ import { sendPhoneCode, sendEmailCode } from '@/service/utils/sendNote';
 import { UserAuthTypeEnum } from '@/constants/common';
 import { customAlphabet } from 'nanoid';
 const nanoid = customAlphabet('123456789', 6);
+import { authGoogleToken } from '@/utils/plugin/google';
+import requestIp from 'request-ip';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    const { username, type } = req.query as { username: string; type: `${UserAuthTypeEnum}` };
+    const { username, type, googleToken } = req.body as {
+      username: string;
+      type: `${UserAuthTypeEnum}`;
+      googleToken: string;
+    };
 
     if (!username || !type) {
       throw new Error('缺少参数');
     }
 
+    // google auth
+    process.env.SERVICE_GOOGLE_VER_TOKEN &&
+      (await authGoogleToken({
+        secret: process.env.SERVICE_GOOGLE_VER_TOKEN,
+        response: googleToken,
+        remoteip: requestIp.getClientIp(req) || undefined
+      }));
+
     await connectToDatabase();
 
     const code = nanoid();

src/pages/chat/index.tsx

@@ -291,7 +291,7 @@ const Chat = ({ modelId, chatId }: { modelId: string; chatId: string }) => {
     }, 100);
 
     try {
-      await gptChatPrompt(newChatList.slice(-2));
+      await gptChatPrompt(newChatList.slice(newChatList.length - 2));
     } catch (err: any) {
       toast({
         title: typeof err === 'string' ? err : err?.message || '聊天出错了~',

src/pages/index.tsx

@@ -2,8 +2,6 @@ import React, { useEffect } from 'react';
 import { Card, Box, Link, Flex, Image, Button } from '@chakra-ui/react';
 import Markdown from '@/components/Markdown';
 import { useMarkdown } from '@/hooks/useMarkdown';
-import { getFilling } from '@/api/system';
-import { useQuery } from '@tanstack/react-query';
 import { useRouter } from 'next/router';
 import { useGlobalStore } from '@/store/global';
 
@@ -13,7 +11,10 @@ const Home = () => {
   const router = useRouter();
   const { inviterId } = router.query as { inviterId: string };
   const { data } = useMarkdown({ url: '/intro.md' });
-  const { isPc } = useGlobalStore();
+  const {
+    isPc,
+    initData: { beianText }
+  } = useGlobalStore();
 
   useEffect(() => {
     if (inviterId) {
@@ -21,8 +22,6 @@ const Home = () => {
     }
   }, [inviterId]);
 
-  const { data: { beianText = '' } = {} } = useQuery(['init'], getFilling);
-
   /* 加载动画 */
   useEffect(() => {
     setTimeout(() => {

src/pages/kb/components/DataCard.tsx

@@ -22,7 +22,6 @@ import {
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import type { BoxProps } from '@chakra-ui/react';
 import type { KbDataItemType } from '@/types/plugin';
-import { ModelDataStatusMap } from '@/constants/model';
 import { usePagination } from '@/hooks/usePagination';
 import {
   getKbDataList,
@@ -91,9 +90,9 @@ const DataCard = ({ kbId }: { kbId: string }) => {
     onClose: onCloseSelectCsvModal
   } = useDisclosure();
 
-  const { data: { splitDataQueue = 0, embeddingQueue = 0 } = {}, refetch } = useQuery(
+  const { data: { qaListLen = 0, vectorListLen = 0 } = {}, refetch } = useQuery(
-    ['getModelSplitDataList'],
+    ['getModelSplitDataList', kbId],
-    () => getTrainingData(kbId),
+    () => getTrainingData({ kbId, init: false }),
     {
       onError(err) {
         console.log(err);
@@ -113,7 +112,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
   // interval get data
   useQuery(['refetchData'], () => refetchData(pageNum), {
     refetchInterval: 5000,
-    enabled: splitDataQueue > 0 || embeddingQueue > 0
+    enabled: qaListLen > 0 || vectorListLen > 0
   });
 
   // get al data and export csv
@@ -161,7 +160,10 @@ const DataCard = ({ kbId }: { kbId: string }) => {
           variant={'outline'}
           mr={[2, 4]}
           size={'sm'}
-          onClick={() => refetchData(pageNum)}
+          onClick={() => {
+            refetchData(pageNum);
+            getTrainingData({ kbId, init: true });
+          }}
         />
         <Button
           variant={'outline'}
@@ -194,10 +196,10 @@ const DataCard = ({ kbId }: { kbId: string }) => {
         </Menu>
       </Flex>
       <Flex mt={4}>
-        {(splitDataQueue > 0 || embeddingQueue > 0) && (
+        {(qaListLen > 0 || vectorListLen > 0) && (
           <Box fontSize={'xs'}>
-            {splitDataQueue > 0 ? `${splitDataQueue}条数据正在拆分，` : ''}
+            {qaListLen > 0 ? `${qaListLen}条数据正在拆分，` : ''}
-            {embeddingQueue > 0 ? `${embeddingQueue}条数据正在生成索引，` : ''}
+            {vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引，` : ''}
             请耐心等待...
           </Box>
         )}
@@ -237,7 +239,6 @@ const DataCard = ({ kbId }: { kbId: string }) => {
                 </Tooltip>
               </Th>
               <Th>补充知识</Th>
-              <Th>状态</Th>
               <Th>操作</Th>
             </Tr>
           </Thead>
@@ -250,7 +251,6 @@ const DataCard = ({ kbId }: { kbId: string }) => {
                 <Td>
                   <Box {...tdStyles.current}>{item.a || '-'}</Box>
                 </Td>
-                <Td>{ModelDataStatusMap[item.status]}</Td>
                 <Td>
                   <IconButton
                     mr={5}

src/pages/kb/components/Detail.tsx

@@ -56,13 +56,13 @@ const Detail = ({ kbId }: { kbId: string }) => {
       }
     },
     onError(err: any) {
+      loadKbList(true);
+      setLastKbId('');
+      router.replace(`/kb`);
       toast({
         title: getErrText(err, '获取知识库异常'),
         status: 'error'
       });
-      loadKbList(true);
-      setLastKbId('');
-      router.replace(`/kb?kbId=${myKbList[0]?._id || ''}`);
     }
   });
 

src/pages/kb/components/InputDataModal.tsx

@@ -13,6 +13,8 @@ import {
 import { useForm } from 'react-hook-form';
 import { postKbDataFromList, putKbDataById } from '@/api/plugins/kb';
 import { useToast } from '@/hooks/useToast';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { getErrText } from '@/utils/tools';
 
 export type FormData = { dataId?: string; a: string; q: string };
 
@@ -52,31 +54,39 @@ const InputDataModal = ({
       setLoading(true);
 
       try {
-        const res = await postKbDataFromList({
+        const { insertLen } = await postKbDataFromList({
           kbId,
           data: [
             {
               a: e.a,
               q: e.q
             }
-          ]
+          ],
+          mode: TrainingModeEnum.index
         });
 
-        toast({
+        if (insertLen === 0) {
-          title: res === 0 ? '可能已存在完全一致的数据' : '导入数据成功,需要一段时间训练',
+          toast({
-          status: 'success'
+            title: '已存在完全一致的数据',
-        });
+            status: 'warning'
-        reset({
+          });
-          a: '',
+        } else {
-          q: ''
+          toast({
-        });
+            title: '导入数据成功,需要一段时间训练',
+            status: 'success'
+          });
+          reset({
+            a: '',
+            q: ''
+          });
+        }
+
         onSuccess();
       } catch (err: any) {
         toast({
-          title: err?.message || '出现了点意外~',
+          title: getErrText(err, '出现了点意外~'),
           status: 'error'
         });
-        console.log(err);
       }
       setLoading(false);
     },
@@ -124,7 +134,8 @@ const InputDataModal = ({
         <ModalCloseButton />
 
         <Box
-          display={['block', 'flex']}
+          display={'flex'}
+          flexDirection={['column', 'row']}
           flex={'1 0 0'}
           h={['100%', 0]}
           overflow={'overlay'}

src/pages/kb/components/SelectCsvModal.tsx

@@ -19,6 +19,8 @@ import { postKbDataFromList } from '@/api/plugins/kb';
 import Markdown from '@/components/Markdown';
 import { useMarkdown } from '@/hooks/useMarkdown';
 import { fileDownload } from '@/utils/file';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { getErrText } from '@/utils/tools';
 
 const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
 
@@ -35,6 +37,7 @@ const SelectJsonModal = ({
   const { toast } = useToast();
   const { File, onOpen } = useSelectFile({ fileType: '.csv', multiple: false });
   const [fileData, setFileData] = useState<{ q: string; a: string }[]>([]);
+  const [successData, setSuccessData] = useState(0);
   const { openConfirm, ConfirmChild } = useConfirm({
     content: '确认导入该数据集?'
   });
@@ -55,9 +58,8 @@ const SelectJsonModal = ({
           }))
         );
       } catch (error: any) {
-        console.log(error);
         toast({
-          title: error?.message || 'csv 文件格式有误',
+          title: getErrText(error, 'csv 文件格式有误'),
           status: 'error'
         });
       }
@@ -66,26 +68,35 @@ const SelectJsonModal = ({
     [setSelecting, toast]
   );
 
-  const { mutate, isLoading } = useMutation({
+  const { mutate, isLoading: uploading } = useMutation({
     mutationFn: async () => {
       if (!fileData || fileData.length === 0) return;
 
-      const res = await postKbDataFromList({
+      let success = 0;
-        kbId,
+
-        data: fileData
+      // subsection import
-      });
+      const step = 50;
+      for (let i = 0; i < fileData.length; i += step) {
+        const { insertLen } = await postKbDataFromList({
+          kbId,
+          data: fileData.slice(i, i + step),
+          mode: TrainingModeEnum.index
+        });
+        success += insertLen || 0;
+        setSuccessData((state) => state + step);
+      }
 
       toast({
-        title: `导入数据成功，最终导入: ${res || 0} 条数据。需要一段时间训练`,
+        title: `导入数据成功，最终导入: ${success} 条数据。需要一段时间训练`,
         status: 'success',
         duration: 4000
       });
       onClose();
       onSuccess();
     },
-    onError() {
+    onError(err) {
       toast({
-        title: '导入文件失败',
+        title: getErrText(err, '导入文件失败'),
         status: 'error'
       });
     }
@@ -119,15 +130,15 @@ const SelectJsonModal = ({
               点击下载csv模板
             </Box>
             <Flex alignItems={'center'}>
-              <Button isLoading={selecting} onClick={onOpen}>
+              <Button isLoading={selecting} isDisabled={uploading} onClick={onOpen}>
                 选择 csv 问答对
               </Button>
 
-              <Box ml={4}>一共 {fileData.length} 组数据</Box>
+              <Box ml={4}>一共 {fileData.length} 组数据（下面最多展示100组）</Box>
             </Flex>
           </Box>
           <Box flex={'3 0 0'} h={'100%'} overflow={'auto'} p={2} backgroundColor={'blackAlpha.50'}>
-            {fileData.map((item, index) => (
+            {fileData.slice(0, 100).map((item, index) => (
               <Box key={index}>
                 <Box>
                   Q{index + 1}. {item.q}
@@ -142,15 +153,15 @@ const SelectJsonModal = ({
 
         <Flex px={6} pt={2} pb={4}>
           <Box flex={1}></Box>
-          <Button variant={'outline'} mr={3} onClick={onClose}>
+          <Button variant={'outline'} isLoading={uploading} mr={3} onClick={onClose}>
             取消
           </Button>
-          <Button
+          <Button isDisabled={fileData.length === 0 || uploading} onClick={openConfirm(mutate)}>
-            isLoading={isLoading}
+            {uploading ? (
-            isDisabled={fileData.length === 0}
+              <Box>{Math.round((successData / fileData.length) * 100)}%</Box>
-            onClick={openConfirm(mutate)}
+            ) : (
-          >
+              '确认导入'
-            确认导入
+            )}
           </Button>
         </Flex>
       </ModalContent>

src/pages/kb/components/SelectFileModal.tsx

@@ -17,21 +17,22 @@ import { useSelectFile } from '@/hooks/useSelectFile';
 import { useConfirm } from '@/hooks/useConfirm';
 import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
 import { useMutation } from '@tanstack/react-query';
-import { postSplitData } from '@/api/plugins/kb';
+import { postKbDataFromList } from '@/api/plugins/kb';
 import Radio from '@/components/Radio';
 import { splitText_token } from '@/utils/file';
-import { SplitTextTypEnum } from '@/constants/plugin';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { getErrText } from '@/utils/tools';
 
 const fileExtension = '.txt,.doc,.docx,.pdf,.md';
 
 const modeMap = {
-  qa: {
+  [TrainingModeEnum.qa]: {
     maxLen: 2800,
     slideLen: 800,
     price: 4,
     isPrompt: true
   },
-  subsection: {
+  [TrainingModeEnum.index]: {
     maxLen: 800,
     slideLen: 300,
     price: 0.4,
@@ -52,11 +53,16 @@ const SelectFileModal = ({
   const { toast } = useToast();
   const [prompt, setPrompt] = useState('');
   const { File, onOpen } = useSelectFile({ fileType: fileExtension, multiple: true });
-  const [mode, setMode] = useState<`${SplitTextTypEnum}`>(SplitTextTypEnum.subsection);
+  const [mode, setMode] = useState<`${TrainingModeEnum}`>(TrainingModeEnum.index);
   const [fileTextArr, setFileTextArr] = useState<string[]>(['']);
-  const [splitRes, setSplitRes] = useState<{ tokens: number; chunks: string[] }>({
+  const [splitRes, setSplitRes] = useState<{
+    tokens: number;
+    chunks: string[];
+    successChunks: number;
+  }>({
     tokens: 0,
-    chunks: []
+    chunks: [],
+    successChunks: 0
   });
   const { openConfirm, ConfirmChild } = useConfirm({
     content: `确认导入该文件，需要一定时间进行拆解，该任务无法终止！如果余额不足，未完成的任务会被直接清除。一共 ${
@@ -103,26 +109,38 @@ const SelectFileModal = ({
     [toast]
   );
 
-  const { mutate, isLoading } = useMutation({
+  const { mutate, isLoading: uploading } = useMutation({
     mutationFn: async () => {
       if (splitRes.chunks.length === 0) return;
 
-      await postSplitData({
+      // subsection import
-        kbId,
+      let success = 0;
-        chunks: splitRes.chunks,
+      const step = 50;
-        prompt: `下面是"${prompt || '一段长文本'}"`,
+      for (let i = 0; i < splitRes.chunks.length; i += step) {
-        mode
+        const { insertLen } = await postKbDataFromList({
-      });
+          kbId,
+          data: splitRes.chunks.slice(i, i + step).map((text) => ({ q: text, a: '' })),
+          prompt: `下面是"${prompt || '一段长文本'}"`,
+          mode
+        });
+
+        success += insertLen;
+        setSplitRes((state) => ({
+          ...state,
+          successChunks: state.successChunks + step
+        }));
+      }
+
       toast({
-        title: '导入数据成功,需要一段拆解和训练',
+        title: `去重后共导入 ${success} 条数据,需要一段拆解和训练.`,
         status: 'success'
       });
       onClose();
       onSuccess();
     },
-    onError() {
+    onError(err) {
       toast({
-        title: '导入文件失败',
+        title: getErrText(err, '导入文件失败'),
         status: 'error'
       });
     }
@@ -130,27 +148,36 @@ const SelectFileModal = ({
 
   const onclickImport = useCallback(async () => {
     setBtnLoading(true);
-    let promise = Promise.resolve();
+    try {
+      let promise = Promise.resolve();
 
-    const splitRes = fileTextArr
+      const splitRes = await Promise.all(
-      .filter((item) => item)
+        fileTextArr
-      .map((item) =>
+          .filter((item) => item)
-        splitText_token({
+          .map((item) =>
-          text: item,
+            splitText_token({
-          ...modeMap[mode]
+              text: item,
-        })
+              ...modeMap[mode]
+            })
+          )
       );
 
-    setSplitRes({
+      setSplitRes({
-      tokens: splitRes.reduce((sum, item) => sum + item.tokens, 0),
+        tokens: splitRes.reduce((sum, item) => sum + item.tokens, 0),
-      chunks: splitRes.map((item) => item.chunks).flat()
+        chunks: splitRes.map((item) => item.chunks).flat(),
-    });
+        successChunks: 0
+      });
 
+      await promise;
+      openConfirm(mutate)();
+    } catch (error) {
+      toast({
+        status: 'warning',
+        title: getErrText(error, '拆分文本异常')
+      });
+    }
     setBtnLoading(false);
-
+  }, [fileTextArr, mode, mutate, openConfirm, toast]);
-    await promise;
-    openConfirm(mutate)();
-  }, [fileTextArr, mode, mutate, openConfirm]);
 
   return (
     <Modal isOpen={true} onClose={onClose} isCentered>
@@ -185,11 +212,11 @@ const SelectFileModal = ({
             <Radio
               ml={3}
               list={[
-                { label: '直接分段', value: 'subsection' },
+                { label: '直接分段', value: 'index' },
                 { label: 'QA拆分', value: 'qa' }
               ]}
               value={mode}
-              onChange={(e) => setMode(e as 'subsection' | 'qa')}
+              onChange={(e) => setMode(e as 'index' | 'qa')}
             />
           </Flex>
           {/* 内容介绍 */}
@@ -225,6 +252,11 @@ const SelectFileModal = ({
                       ...fileTextArr.slice(i + 1)
                     ]);
                   }}
+                  onBlur={(e) => {
+                    if (fileTextArr.length > 1 && e.target.value === '') {
+                      setFileTextArr((state) => [...state.slice(0, i), ...state.slice(i + 1)]);
+                    }
+                  }}
                 />
               </Box>
             ))}
@@ -232,19 +264,22 @@ const SelectFileModal = ({
         </ModalBody>
 
         <Flex px={6} pt={2} pb={4}>
-          <Button isLoading={btnLoading} onClick={onOpen}>
+          <Button isLoading={btnLoading} isDisabled={uploading} onClick={onOpen}>
             选择文件
           </Button>
           <Box flex={1}></Box>
-          <Button variant={'outline'} colorScheme={'gray'} mr={3} onClick={onClose}>
+          <Button variant={'outline'} isLoading={uploading} mr={3} onClick={onClose}>
             取消
           </Button>
           <Button
-            isLoading={isLoading || btnLoading}
+            isDisabled={uploading || btnLoading || fileTextArr[0] === ''}
-            isDisabled={isLoading || btnLoading || fileTextArr[0] === ''}
             onClick={onclickImport}
           >
-            确认导入
+            {uploading ? (
+              <Box>{Math.round((splitRes.successChunks / splitRes.chunks.length) * 100)}%</Box>
+            ) : (
+              '确认导入'
+            )}
           </Button>
         </Flex>
       </ModalContent>

src/service/api/request.ts

@@ -53,10 +53,11 @@ function responseError(err: any) {
 }
 
 /* 创建请求实例 */
-const instance = axios.create({
+export const instance = axios.create({
   timeout: 60000, // 超时时间
+  baseURL: `http://localhost:${process.env.PORT || 3000}/api`,
   headers: {
-    'content-type': 'application/json'
+    rootkey: process.env.ROOT_KEY
   }
 });
 
@@ -75,7 +76,6 @@ function request(url: string, data: any, config: ConfigType, method: Method): an
 
   return instance
     .request({
-      baseURL: `http://localhost:${process.env.PORT || 3000}/api`,
       url,
       method,
       data: method === 'GET' ? null : data,
@@ -93,18 +93,30 @@ function request(url: string, data: any, config: ConfigType, method: Method): an
  * @param {Object} config
  * @returns
  */
-export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+export function GET<T = { data: any }>(
+  url: string,
+  params = {},
+  config: ConfigType = {}
+): Promise<T> {
   return request(url, params, config, 'GET');
 }
 
-export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function POST<T = { data: any }>(
+  url: string,
+  data = {},
+  config: ConfigType = {}
+): Promise<T> {
   return request(url, data, config, 'POST');
 }
 
-export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function PUT<T = { data: any }>(
+  url: string,
+  data = {},
+  config: ConfigType = {}
+): Promise<T> {
   return request(url, data, config, 'PUT');
 }
 
-export function DELETE<T>(url: string, config: ConfigType = {}): Promise<T> {
+export function DELETE<T = { data: any }>(url: string, config: ConfigType = {}): Promise<T> {
   return request(url, {}, config, 'DELETE');
 }

src/service/errorCode.ts

@@ -24,10 +24,10 @@ export const openaiError: Record<string, string> = {
   'Bad Request': 'Bad Request~ 可能内容太多了',
   'Bad Gateway': '网关异常，请重试'
 };
-export const openaiError2: Record<string, string> = {
+export const openaiAccountError: Record<string, string> = {
   insufficient_quota: 'API 余额不足',
-  billing_not_active: 'openai 账号异常',
+  invalid_api_key: 'openai 账号异常',
-  invalid_request_error: '无效的 openai 请求'
+  account_deactivated: '账号已停用'
 };
 export const proxyError: Record<string, boolean> = {
   ECONNABORTED: true,

src/service/events/generateQA.ts

@@ -1,75 +1,91 @@
-import { SplitData } from '@/service/mongo';
+import { TrainingData } from '@/service/mongo';
 import { getApiKey } from '../utils/auth';
 import { OpenAiChatEnum } from '@/constants/model';
 import { pushSplitDataBill } from '@/service/events/pushBill';
-import { generateVector } from './generateVector';
+import { openaiAccountError } from '../errorCode';
-import { openaiError2 } from '../errorCode';
-import { insertKbItem } from '@/service/pg';
-import { SplitDataSchema } from '@/types/mongoSchema';
 import { modelServiceToolMap } from '../utils/chat';
 import { ChatRoleEnum } from '@/constants/chat';
-import { getErrText } from '@/utils/tools';
 import { BillTypeEnum } from '@/constants/user';
+import { pushDataToKb } from '@/pages/api/openapi/kb/pushData';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { ERROR_ENUM } from '../errorCode';
 
-export async function generateQA(next = false): Promise<any> {
+const reduceQueue = () => {
-  if (process.env.queueTask !== '1') {
+  global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
-    try {
+};
-      fetch(process.env.parentUrl || '');
-    } catch (error) {
-      console.log('parentUrl fetch error', error);
-    }
-    return;
-  }
-  if (global.generatingQA === true && !next) return;
 
-  global.generatingQA = true;
+export async function generateQA(): Promise<any> {
+  const maxProcess = Number(process.env.QA_MAX_PROCESS || 10);
 
-  let dataId = null;
+  if (global.qaQueueLen >= maxProcess) return;
+  global.qaQueueLen++;
+
+  let trainingId = '';
+  let userId = '';
 
   try {
-    // 找出一个需要生成的 dataItem
+    const match = {
-    const data = await SplitData.aggregate([
+      mode: TrainingModeEnum.qa,
-      { $match: { textList: { $exists: true, $ne: [] } } },
+      lockTime: { $lte: new Date(Date.now() - 4 * 60 * 1000) }
-      { $sample: { size: 1 } }
+    };
+    // random get task
+    const agree = await TrainingData.aggregate([
+      {
+        $match: match
+      },
+      { $sample: { size: 1 } },
+      {
+        $project: {
+          _id: 1
+        }
+      }
     ]);
 
-    const dataItem: SplitDataSchema = data[0];
+    // no task
-
+    if (agree.length === 0) {
-    if (!dataItem) {
+      reduceQueue();
-      console.log('没有需要生成 QA 的数据');
+      global.qaQueueLen <= 0 && console.log(`没有需要【QA】的数据, ${global.qaQueueLen}`);
-      global.generatingQA = false;
       return;
     }
 
-    dataId = dataItem._id;
+    const data = await TrainingData.findOneAndUpdate(
+      {
+        _id: agree[0]._id,
+        ...match
+      },
+      {
+        lockTime: new Date()
+      }
+    ).select({
+      _id: 1,
+      userId: 1,
+      kbId: 1,
+      prompt: 1,
+      q: 1
+    });
 
-    // 获取 5 个源文本
+    // task preemption
-    const textList: string[] = dataItem.textList.slice(-5);
+    if (!data) {
-
+      reduceQueue();
-    // 获取 openapi Key
+      return generateQA();
-    let userOpenAiKey = '',
-      systemAuthKey = '';
-    try {
-      const key = await getApiKey({ model: OpenAiChatEnum.GPT35, userId: dataItem.userId });
-      userOpenAiKey = key.userOpenAiKey;
-      systemAuthKey = key.systemAuthKey;
-    } catch (err: any) {
-      // 余额不够了, 清空该记录
-      await SplitData.findByIdAndUpdate(dataItem._id, {
-        textList: [],
-        errorText: getErrText(err, '获取 OpenAi Key 失败')
-      });
-      generateQA(true);
-      return;
     }
 
-    console.log(`正在生成一组QA, 包含 ${textList.length} 组文本。ID: ${dataItem._id}`);
+    trainingId = data._id;
+    userId = String(data.userId);
+    const kbId = String(data.kbId);
+
+    // 余额校验并获取 openapi Key
+    const { userOpenAiKey, systemAuthKey } = await getApiKey({
+      model: OpenAiChatEnum.GPT35,
+      userId,
+      type: 'training'
+    });
 
     const startTime = Date.now();
 
     // 请求 chatgpt 获取回答
-    const response = await Promise.allSettled(
+    const response = await Promise.all(
-      textList.map((text) =>
+      [data.q].map((text) =>
         modelServiceToolMap[OpenAiChatEnum.GPT35]
           .chatCompletion({
             apiKey: userOpenAiKey || systemAuthKey,
@@ -78,7 +94,7 @@ export async function generateQA(next = false): Promise<any> {
               {
                 obj: ChatRoleEnum.System,
                 value: `你是出题人
-${dataItem.prompt || '下面是"一段长文本"'}
+${data.prompt || '下面是"一段长文本"'}
 从中选出5至20个题目和答案.答案详细.按格式返回: Q1:
 A1:
 Q2:
@@ -98,7 +114,7 @@ A2:
             // 计费
             pushSplitDataBill({
               isPay: !userOpenAiKey && result.length > 0,
-              userId: dataItem.userId,
+              userId: data.userId,
               type: BillTypeEnum.QA,
               textLen: responseMessages.map((item) => item.value).join('').length,
               totalTokens
@@ -116,57 +132,58 @@ A2:
       )
     );
 
-    // 获取成功的回答
+    const responseList = response.map((item) => item.result).flat();
-    const successResponse: {
-      rawContent: string;
-      result: {
-        q: string;
-        a: string;
-      }[];
-    }[] = response.filter((item) => item.status === 'fulfilled').map((item: any) => item.value);
 
-    const resultList = successResponse.map((item) => item.result).flat();
+    // 创建 向量生成 队列
+    await pushDataToKb({
+      kbId,
+      data: responseList,
+      userId,
+      mode: TrainingModeEnum.index
+    });
+
+    // delete data from training
+    await TrainingData.findByIdAndDelete(data._id);
 
-    await Promise.allSettled([
-      // 删掉后5个数据
-      SplitData.findByIdAndUpdate(dataItem._id, {
-        textList: dataItem.textList.slice(0, -5)
-      }),
-      // 生成的内容插入 pg
-      insertKbItem({
-        userId: dataItem.userId,
-        kbId: dataItem.kbId,
-        data: resultList
-      })
-    ]);
     console.log('生成QA成功，time:', `${(Date.now() - startTime) / 1000}s`);
 
-    generateQA(true);
+    reduceQueue();
-    generateVector();
+    generateQA();
-  } catch (error: any) {
+  } catch (err: any) {
+    reduceQueue();
     // log
-    if (error?.response) {
+    if (err?.response) {
       console.log('openai error: 生成QA错误');
-      console.log(error.response?.status, error.response?.statusText, error.response?.data);
+      console.log(err.response?.status, err.response?.statusText, err.response?.data);
     } else {
-      console.log('生成QA错误:', error);
+      console.log('生成QA错误:', err);
     }
 
-    // 没有余额或者凭证错误时，拒绝任务
+    // message error or openai account error
-    if (dataId && openaiError2[error?.response?.data?.error?.type]) {
+    if (
-      console.log(openaiError2[error?.response?.data?.error?.type], '删除QA任务');
+      err?.message === 'invalid message format' ||
+      err.response?.statusText === 'Unauthorized' ||
+      openaiAccountError[err?.response?.data?.error?.code || err?.response?.data?.error?.type]
+    ) {
+      await TrainingData.findByIdAndRemove(trainingId);
+    }
 
-      await SplitData.findByIdAndUpdate(dataId, {
+    // 账号余额不足，删除任务
-        textList: [],
+    if (err === ERROR_ENUM.insufficientQuota) {
-        errorText: 'api 余额不足'
+      console.log('余额不足，删除向量生成任务');
+      await TrainingData.deleteMany({
+        userId
       });
-
+      return generateQA();
-      generateQA(true);
-      return;
     }
 
+    // unlock
+    await TrainingData.findByIdAndUpdate(trainingId, {
+      lockTime: new Date('2000/1/1')
+    });
+
     setTimeout(() => {
-      generateQA(true);
+      generateQA();
     }, 1000);
   }
 }

src/service/events/generateVector.ts

@@ -1,107 +1,141 @@
-import { getApiKey } from '../utils/auth';
+import { openaiAccountError } from '../errorCode';
-import { openaiError2 } from '../errorCode';
+import { insertKbItem } from '@/service/pg';
-import { PgClient } from '@/service/pg';
-import { getErrText } from '@/utils/tools';
 import { openaiEmbedding } from '@/pages/api/openapi/plugin/openaiEmbedding';
+import { TrainingData } from '../models/trainingData';
+import { ERROR_ENUM } from '../errorCode';
+import { TrainingModeEnum } from '@/constants/plugin';
 
-export async function generateVector(next = false): Promise<any> {
+const reduceQueue = () => {
-  if (process.env.queueTask !== '1') {
+  global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0;
-    try {
+};
-      fetch(process.env.parentUrl || '');
-    } catch (error) {
-      console.log('parentUrl fetch error', error);
-    }
-    return;
-  }
 
-  if (global.generatingVector && !next) return;
+/* 索引生成队列。每导入一次，就是一个单独的线程 */
+export async function generateVector(): Promise<any> {
+  const maxProcess = Number(process.env.VECTOR_MAX_PROCESS || 10);
 
-  global.generatingVector = true;
+  if (global.vectorQueueLen >= maxProcess) return;
-  let dataId = null;
+  global.vectorQueueLen++;
+
+  let trainingId = '';
+  let userId = '';
 
   try {
-    // 从找出一个 status = waiting 的数据
+    const match = {
-    const searchRes = await PgClient.select('modelData', {
+      mode: TrainingModeEnum.index,
-      fields: ['id', 'q', 'user_id'],
+      lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) }
-      where: [['status', 'waiting']],
+    };
-      limit: 1
+    // random get task
-    });
+    const agree = await TrainingData.aggregate([
+      {
+        $match: match
+      },
+      { $sample: { size: 1 } },
+      {
+        $project: {
+          _id: 1
+        }
+      }
+    ]);
 
-    if (searchRes.rowCount === 0) {
+    // no task
-      console.log('没有需要生成 【向量】 的数据');
+    if (agree.length === 0) {
-      global.generatingVector = false;
+      reduceQueue();
+      global.vectorQueueLen <= 0 && console.log(`没有需要【索引】的数据, ${global.vectorQueueLen}`);
       return;
     }
 
-    const dataItem: { id: string; q: string; userId: string } = {
+    const data = await TrainingData.findOneAndUpdate(
-      id: searchRes.rows[0].id,
+      {
-      q: searchRes.rows[0].q,
+        _id: agree[0]._id,
-      userId: searchRes.rows[0].user_id
+        ...match
-    };
+      },
+      {
+        lockTime: new Date()
+      }
+    ).select({
+      _id: 1,
+      userId: 1,
+      kbId: 1,
+      q: 1,
+      a: 1
+    });
 
-    dataId = dataItem.id;
+    // task preemption
-
+    if (!data) {
-    // 获取 openapi Key
+      reduceQueue();
-    try {
+      return generateVector();
-      await getApiKey({ model: 'gpt-3.5-turbo', userId: dataItem.userId });
-    } catch (err: any) {
-      await PgClient.delete('modelData', {
-        where: [['id', dataId]]
-      });
-      getErrText(err, '获取 OpenAi Key 失败');
-      return generateVector(true);
     }
 
+    trainingId = data._id;
+    userId = String(data.userId);
+    const kbId = String(data.kbId);
+
+    const dataItems = [
+      {
+        q: data.q,
+        a: data.a
+      }
+    ];
+
     // 生成词向量
     const vectors = await openaiEmbedding({
-      input: [dataItem.q],
+      input: dataItems.map((item) => item.q),
-      userId: dataItem.userId
+      userId,
+      type: 'training'
     });
 
-    // 更新 pg 向量和状态数据
+    // 生成结果插入到 pg
-    await PgClient.update('modelData', {
+    await insertKbItem({
-      values: [
+      userId,
-        { key: 'vector', value: `[${vectors[0]}]` },
+      kbId,
-        { key: 'status', value: `ready` }
+      data: vectors.map((vector, i) => ({
-      ],
+        q: dataItems[i].q,
-      where: [['id', dataId]]
+        a: dataItems[i].a,
+        vector
+      }))
     });
 
-    console.log(`生成向量成功: ${dataItem.id}`);
+    // delete data from training
+    await TrainingData.findByIdAndDelete(data._id);
+    console.log(`生成向量成功: ${data._id}`);
 
-    generateVector(true);
+    reduceQueue();
-  } catch (error: any) {
+    generateVector();
+  } catch (err: any) {
+    reduceQueue();
     // log
-    if (error?.response) {
+    if (err?.response) {
       console.log('openai error: 生成向量错误');
-      console.log(error.response?.status, error.response?.statusText, error.response?.data);
+      console.log(err.response?.status, err.response?.statusText, err.response?.data);
     } else {
-      console.log('生成向量错误:', error);
+      console.log('生成向量错误:', err);
+    }
+    // message error or openai account error
+    if (
+      err?.message === 'invalid message format' ||
+      err.response?.statusText === 'Unauthorized' ||
+      openaiAccountError[err?.response?.data?.error?.code || err?.response?.data?.error?.type]
+    ) {
+      console.log('删除一个任务');
+      await TrainingData.findByIdAndRemove(trainingId);
     }
 
-    // 没有余额或者凭证错误时，拒绝任务
+    // 账号余额不足，删除任务
-    if (dataId && openaiError2[error?.response?.data?.error?.type]) {
+    if (err === ERROR_ENUM.insufficientQuota) {
-      console.log('删除向量生成任务记录');
+      console.log('余额不足，删除向量生成任务');
-      try {
+      await TrainingData.deleteMany({
-        await PgClient.delete('modelData', {
+        userId
-          where: [['id', dataId]]
+      });
-        });
+      return generateVector();
-      } catch (error) {
-        error;
-      }
-      generateVector(true);
-      return;
-    }
-    if (error?.response?.statusText === 'Too Many Requests') {
-      console.log('生成向量次数限制，1分钟后尝试');
-      // 限制次数，1分钟后再试
-      setTimeout(() => {
-        generateVector(true);
-      }, 60000);
-      return;
     }
+
+    // unlock
+    await TrainingData.findByIdAndUpdate(trainingId, {
+      lockTime: new Date('2000/1/1')
+    });
+
     setTimeout(() => {
-      generateVector(true);
+      generateVector();
     }, 1000);
   }
 }

src/service/events/pushBill.ts

@@ -134,9 +134,9 @@ export const pushGenerateVectorBill = async ({
   text: string;
   tokenLen: number;
 }) => {
-  console.log(
+  // console.log(
-    `vector generate success. text len: ${text.length}. token len: ${tokenLen}. pay:${isPay}`
+  //   `vector generate success. text len: ${text.length}. token len: ${tokenLen}. pay:${isPay}`
-  );
+  // );
   if (!isPay) return;
 
   let billId;

src/service/models/chat.ts

@@ -50,22 +50,10 @@ const ChatSchema = new Schema({
         quote: {
           type: [
             {
-              id: {
+              id: String,
-                type: String,
+              q: String,
-                required: true
+              a: String,
-              },
+              isEdit: Boolean
-              q: {
-                type: String,
-                default: ''
-              },
-              a: {
-                type: String,
-                default: ''
-              },
-              isEdit: {
-                type: String,
-                default: false
-              }
             }
           ],
           default: []

src/service/models/model.ts

@@ -2,7 +2,7 @@ import { Schema, model, models, Model as MongoModel } from 'mongoose';
 import { ModelSchema as ModelType } from '@/types/mongoSchema';
 import {
   ModelVectorSearchModeMap,
-  ModelVectorSearchModeEnum,
+  appVectorSearchModeEnum,
   ChatModelMap,
   OpenAiChatEnum
 } from '@/constants/model';
@@ -40,7 +40,7 @@ const ModelSchema = new Schema({
       // knowledge base search mode
       type: String,
       enum: Object.keys(ModelVectorSearchModeMap),
-      default: ModelVectorSearchModeEnum.hightSimilarity
+      default: appVectorSearchModeEnum.hightSimilarity
     },
     systemPrompt: {
       // 系统提示词

src/service/models/splitData.ts

@@ -1,32 +0,0 @@
-/* 模型的知识库 */
-import { Schema, model, models, Model as MongoModel } from 'mongoose';
-import { SplitDataSchema as SplitDataType } from '@/types/mongoSchema';
-
-const SplitDataSchema = new Schema({
-  userId: {
-    type: Schema.Types.ObjectId,
-    ref: 'user',
-    required: true
-  },
-  prompt: {
-    // 拆分时的提示词
-    type: String,
-    required: true
-  },
-  kbId: {
-    type: Schema.Types.ObjectId,
-    ref: 'kb',
-    required: true
-  },
-  textList: {
-    type: [String],
-    default: []
-  },
-  errorText: {
-    type: String,
-    default: ''
-  }
-});
-
-export const SplitData: MongoModel<SplitDataType> =
-  models['splitData'] || model('splitData', SplitDataSchema);

src/service/models/trainingData.ts

@@ -0,0 +1,47 @@
+/* 模型的知识库 */
+import { Schema, model, models, Model as MongoModel } from 'mongoose';
+import { TrainingDataSchema as TrainingDateType } from '@/types/mongoSchema';
+import { TrainingTypeMap } from '@/constants/plugin';
+
+// pgList and vectorList, Only one of them will work
+const TrainingDataSchema = new Schema({
+  userId: {
+    type: Schema.Types.ObjectId,
+    ref: 'user',
+    required: true
+  },
+  kbId: {
+    type: Schema.Types.ObjectId,
+    ref: 'kb',
+    required: true
+  },
+  lockTime: {
+    type: Date,
+    default: () => new Date('2000/1/1')
+  },
+  mode: {
+    type: String,
+    enum: Object.keys(TrainingTypeMap),
+    required: true
+  },
+  prompt: {
+    // 拆分时的提示词
+    type: String,
+    default: ''
+  },
+  q: {
+    // 如果是
+    type: String,
+    default: ''
+  },
+  a: {
+    type: String,
+    default: ''
+  },
+  vectorList: {
+    type: Object
+  }
+});
+
+export const TrainingData: MongoModel<TrainingDateType> =
+  models['trainingData'] || model('trainingData', TrainingDataSchema);

src/service/mongo.ts

@@ -1,7 +1,7 @@
 import mongoose from 'mongoose';
-import { generateQA } from './events/generateQA';
-import { generateVector } from './events/generateVector';
 import tunnel from 'tunnel';
+import { TrainingData } from './mongo';
+import { startQueue } from './utils/tools';
 
 /**
  * 连接 MongoDB 数据库
@@ -27,9 +27,6 @@ export async function connectToDatabase(): Promise<void> {
     global.mongodb = null;
   }
 
-  generateQA();
-  generateVector();
-
   // 创建代理对象
   if (process.env.AXIOS_PROXY_HOST && process.env.AXIOS_PROXY_PORT) {
     global.httpsAgent = tunnel.httpsOverHttp({
@@ -39,6 +36,23 @@ export async function connectToDatabase(): Promise<void> {
       }
     });
   }
+
+  global.qaQueueLen = 0;
+  global.vectorQueueLen = 0;
+
+  startQueue();
+  // 5 分钟后解锁不正常的数据，并触发开始训练
+  setTimeout(async () => {
+    await TrainingData.updateMany(
+      {
+        lockTime: { $lte: Date.now() - 5 * 60 * 1000 }
+      },
+      {
+        lockTime: new Date('2000/1/1')
+      }
+    );
+    startQueue();
+  }, 5 * 60 * 1000);
 }
 
 export * from './models/authCode';
@@ -47,7 +61,7 @@ export * from './models/model';
 export * from './models/user';
 export * from './models/bill';
 export * from './models/pay';
-export * from './models/splitData';
+export * from './models/trainingData';
 export * from './models/openapi';
 export * from './models/promotionRecord';
 export * from './models/collection';

src/service/pg.ts

@@ -13,8 +13,8 @@ export const connectPg = async () => {
     password: process.env.PG_PASSWORD,
     database: process.env.PG_DB_NAME,
     max: 20,
-    idleTimeoutMillis: 30000,
+    idleTimeoutMillis: 60000,
-    connectionTimeoutMillis: 2000
+    connectionTimeoutMillis: 20000
   });
 
   global.pgClient.on('error', (err) => {
@@ -168,6 +168,7 @@ export const insertKbItem = ({
   userId: string;
   kbId: string;
   data: {
+    vector: number[];
     q: string;
     a: string;
   }[];
@@ -176,9 +177,9 @@ export const insertKbItem = ({
     values: data.map((item) => [
       { key: 'user_id', value: userId },
       { key: 'kb_id', value: kbId },
-      { key: 'q', value: item.q },
+      { key: 'q', value: item.q.replace(/'/g, '"') },
-      { key: 'a', value: item.a },
+      { key: 'a', value: item.a.replace(/'/g, '"') },
-      { key: 'status', value: 'waiting' }
+      { key: 'vector', value: `[${item.vector}]` }
     ])
   });
 };

src/service/response.ts

@@ -1,5 +1,11 @@
 import { NextApiResponse } from 'next';
-import { openaiError, openaiError2, proxyError, ERROR_RESPONSE, ERROR_ENUM } from './errorCode';
+import {
+  openaiError,
+  openaiAccountError,
+  proxyError,
+  ERROR_RESPONSE,
+  ERROR_ENUM
+} from './errorCode';
 import { clearCookie } from './utils/tools';
 
 export interface ResponseType<T = any> {
@@ -40,8 +46,8 @@ export const jsonRes = <T = any>(
       msg = '接口连接异常';
     } else if (error?.response?.data?.error?.message) {
       msg = error?.response?.data?.error?.message;
-    } else if (openaiError2[error?.response?.data?.error?.type]) {
+    } else if (openaiAccountError[error?.response?.data?.error?.code]) {
-      msg = openaiError2[error?.response?.data?.error?.type];
+      msg = openaiAccountError[error?.response?.data?.error?.code];
     } else if (openaiError[error?.response?.statusText]) {
       msg = openaiError[error.response.statusText];
     }

src/service/utils/auth.ts

@@ -5,12 +5,14 @@ import { Chat, Model, OpenApi, User, ShareChat, KB } from '../mongo';
 import type { ModelSchema } from '@/types/mongoSchema';
 import type { ChatItemSimpleType } from '@/types/chat';
 import mongoose from 'mongoose';
-import { ClaudeEnum, defaultModel } from '@/constants/model';
+import { ClaudeEnum, defaultModel, embeddingModel, EmbeddingModelType } from '@/constants/model';
 import { formatPrice } from '@/utils/user';
 import { ERROR_ENUM } from '../errorCode';
 import { ChatModelType, OpenAiChatEnum } from '@/constants/model';
 import { hashPassword } from '@/service/utils/tools';
 
+export type ApiKeyType = 'training' | 'chat';
+
 export const parseCookie = (cookie?: string): Promise<string> => {
   return new Promise((resolve, reject) => {
     // 获取 cookie
@@ -118,9 +120,15 @@ export const authUser = async ({
 };
 
 /* random get openai api key */
-export const getSystemOpenAiKey = () => {
+export const getSystemOpenAiKey = (type: ApiKeyType) => {
+  const keys = (() => {
+    if (type === 'training') {
+      return process.env.OPENAI_TRAINING_KEY?.split(',') || [];
+    }
+    return process.env.OPENAIKEY?.split(',') || [];
+  })();
+
   // 纯字符串类型
-  const keys = process.env.OPENAIKEY?.split(',') || [];
   const i = Math.floor(Math.random() * keys.length);
   return keys[i] || (process.env.OPENAIKEY as string);
 };
@@ -129,11 +137,13 @@ export const getSystemOpenAiKey = () => {
 export const getApiKey = async ({
   model,
   userId,
-  mustPay = false
+  mustPay = false,
+  type = 'chat'
 }: {
   model: ChatModelType;
   userId: string;
   mustPay?: boolean;
+  type?: ApiKeyType;
 }) => {
   const user = await User.findById(userId);
   if (!user) {
@@ -143,7 +153,7 @@ export const getApiKey = async ({
   const keyMap = {
     [OpenAiChatEnum.GPT35]: {
       userOpenAiKey: user.openaiKey || '',
-      systemAuthKey: getSystemOpenAiKey() as string
+      systemAuthKey: getSystemOpenAiKey(type) as string
     },
     [OpenAiChatEnum.GPT4]: {
       userOpenAiKey: user.openaiKey || '',

src/service/utils/tools.ts

@@ -2,6 +2,8 @@ import type { NextApiResponse, NextApiHandler, NextApiRequest } from 'next';
 import NextCors from 'nextjs-cors';
 import crypto from 'crypto';
 import jwt from 'jsonwebtoken';
+import { generateQA } from '../events/generateQA';
+import { generateVector } from '../events/generateVector';
 
 /* 密码加密 */
 export const hashPassword = (psw: string) => {
@@ -45,7 +47,7 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler {
     req: NextApiRequest,
     res: NextApiResponse
   ) {
-    const methods = ['GET', 'HEAD', 'PUT', 'PATCH', 'POST', 'DELETE'];
+    const methods = ['GET', 'eHEAD', 'PUT', 'PATCH', 'POST', 'DELETE'];
     const origin = req.headers.origin;
     await NextCors(req, res, {
       methods,
@@ -56,3 +58,15 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler {
     return handler(req, res);
   };
 }
+
+export const startQueue = () => {
+  const qaMax = Number(process.env.QA_MAX_PROCESS || 10);
+  const vectorMax = Number(process.env.VECTOR_MAX_PROCESS || 10);
+
+  for (let i = 0; i < qaMax; i++) {
+    generateQA();
+  }
+  for (let i = 0; i < vectorMax; i++) {
+    generateVector();
+  }
+};

src/store/global.ts

@@ -1,8 +1,12 @@
 import { create } from 'zustand';
 import { devtools } from 'zustand/middleware';
 import { immer } from 'zustand/middleware/immer';
+import type { InitDateResponse } from '@/pages/api/system/getInitData';
+import { getInitData } from '@/api/system';
 
 type State = {
+  initData: InitDateResponse;
+  loadInitData: () => Promise<void>;
   loading: boolean;
   setLoading: (val: boolean) => null;
   screenWidth: number;
@@ -13,6 +17,18 @@ type State = {
 export const useGlobalStore = create<State>()(
   devtools(
     immer((set, get) => ({
+      initData: {
+        beianText: '',
+        googleVerKey: ''
+      },
+      async loadInitData() {
+        try {
+          const res = await getInitData();
+          set((state) => {
+            state.initData = res;
+          });
+        } catch (error) {}
+      },
       loading: false,
       setLoading: (val: boolean) => {
         set((state) => {

src/styles/reset.scss

@@ -25,10 +25,6 @@ svg {
   margin: 0;
 }
 
-#__next {
-  height: 100%;
-}
-
 ::-webkit-scrollbar {
   width: 8px;
   height: 8px;
@@ -65,16 +61,26 @@ textarea::placeholder {
   font-size: 0.85em;
 }
 
+* {
+  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
+  -webkit-focus-ring-color: rgba(0, 0, 0, 0);
+  outline: none;
+}
+
+#__next {
+  height: 100%;
+}
+#nprogress .bar {
+  background: '#85b1ff' !important; //自定义颜色
+}
+
 .textEllipsis {
   text-overflow: ellipsis;
   white-space: nowrap;
   overflow: hidden;
 }
-
+.grecaptcha-badge {
-* {
+  display: none !important;
-  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
-  -webkit-focus-ring-color: rgba(0, 0, 0, 0);
-  outline: none;
 }
 
 @media (max-width: 900px) {
@@ -93,7 +99,3 @@ textarea::placeholder {
     padding-bottom: env(safe-area-inset-bottom);
   }
 }
-
-#nprogress .bar {
-  background: '#85b1ff' !important; //自定义颜色
-}

src/types/index.d.ts

@@ -1,18 +1,16 @@
 import type { Mongoose } from 'mongoose';
-import type { RedisClientType } from 'redis';
 import type { Agent } from 'http';
 import type { Pool } from 'pg';
 
 declare global {
   var mongodb: Mongoose | string | null;
-  var redisClient: RedisClientType | null;
   var pgClient: Pool | null;
-  var generatingQA: boolean;
-  var generatingAbstract: boolean;
-  var generatingVector: boolean;
-  var QRCode: any;
   var httpsAgent: Agent;
   var particlesJS: any;
+  var grecaptcha: any;
+  var QRCode: any;
+  var qaQueueLen: number;
+  var vectorQueueLen: number;
 
   interface Window {
     ['pdfjs-dist/build/pdf']: any;

src/types/model.d.ts

@@ -1,6 +1,6 @@
 import { ModelStatusEnum } from '@/constants/model';
 import type { ModelSchema, kbSchema } from './mongoSchema';
-import { ChatModelType, ModelVectorSearchModeEnum } from '@/constants/model';
+import { ChatModelType, appVectorSearchModeEnum } from '@/constants/model';
 
 export type ModelListItemType = {
   _id: string;

src/types/mongoSchema.d.ts

@@ -2,12 +2,13 @@ import type { ChatItemType } from './chat';
 import {
   ModelStatusEnum,
   ModelNameEnum,
-  ModelVectorSearchModeEnum,
+  appVectorSearchModeEnum,
   ChatModelType,
   EmbeddingModelType
 } from '@/constants/model';
 import type { DataType } from './data';
 import { BillTypeEnum } from '@/constants/user';
+import { TrainingModeEnum } from '@/constants/plugin';
 
 export interface UserModelSchema {
   _id: string;
@@ -44,7 +45,7 @@ export interface ModelSchema {
   updateTime: number;
   chat: {
     relatedKbs: string[];
-    searchMode: `${ModelVectorSearchModeEnum}`;
+    searchMode: `${appVectorSearchModeEnum}`;
     systemPrompt: string;
     temperature: number;
     chatModel: ChatModelType; // 聊天时用的模型，训练后就是训练的模型
@@ -68,13 +69,15 @@ export interface CollectionSchema {
 
 export type ModelDataType = 0 | 1;
 
-export interface SplitDataSchema {
+export interface TrainingDataSchema {
   _id: string;
   userId: string;
   kbId: string;
+  lockTime: Date;
+  mode: `${TrainingModeEnum}`;
   prompt: string;
-  errorText: string;
+  q: string;
-  textList: string[];
+  a: string;
 }
 
 export interface ChatSchema {

src/types/pg.d.ts

@@ -1,11 +1,7 @@
-import { ModelDataStatusEnum } from '@/constants/model';
-
 export interface PgKBDataItemType {
   id: string;
   q: string;
   a: string;
-  status: `${ModelDataStatusEnum}`;
+  user_id: string;
-  // model_id: string;
+  kb_id: string;
-  // user_id: string;
-  // kb_id: string;
 }

src/types/plugin.d.ts

@@ -1,5 +1,4 @@
 import type { kbSchema } from './mongoSchema';
-import { PluginTypeEnum } from '@/constants/plugin';
 
 /* kb type */
 export interface KbItemType extends kbSchema {
@@ -9,27 +8,12 @@ export interface KbItemType extends kbSchema {
 
 export interface KbDataItemType {
   id: string;
-  status: 'waiting' | 'ready';
   q: string; // 提问词
   a: string; // 原文
   kbId: string;
   userId: string;
 }
 
-/* plugin */
-export interface PluginConfig {
-  name: string;
-  desc: string;
-  url: string;
-  category: `${PluginTypeEnum}`;
-  uniPrice: 22; // 1k token
-  params: [
-    {
-      type: '';
-    }
-  ];
-}
-
 export type TextPluginRequestParams = {
   input: string;
 };

src/utils/file.ts

@@ -145,7 +145,7 @@ export const fileDownload = ({
  * slideLen - The size of the before and after Text
  * maxLen > slideLen
  */
-export const splitText_token = ({
+export const splitText_token = async ({
   text,
   maxLen,
   slideLen
@@ -154,32 +154,39 @@ export const splitText_token = ({
   maxLen: number;
   slideLen: number;
 }) => {
-  const enc = getOpenAiEncMap()['gpt-3.5-turbo'];
+  try {
-  // filter empty text. encode sentence
+    const enc = getOpenAiEncMap()['gpt-3.5-turbo'];
-  const encodeText = enc.encode(text);
+    // filter empty text. encode sentence
+    const encodeText = enc.encode(text);
 
-  const chunks: string[] = [];
+    const chunks: string[] = [];
-  let tokens = 0;
+    let tokens = 0;
 
-  let startIndex = 0;
+    let startIndex = 0;
-  let endIndex = Math.min(startIndex + maxLen, encodeText.length);
+    let endIndex = Math.min(startIndex + maxLen, encodeText.length);
-  let chunkEncodeArr = encodeText.slice(startIndex, endIndex);
+    let chunkEncodeArr = encodeText.slice(startIndex, endIndex);
 
-  const decoder = new TextDecoder();
+    const decoder = new TextDecoder();
 
-  while (startIndex < encodeText.length) {
+    while (startIndex < encodeText.length) {
-    tokens += chunkEncodeArr.length;
+      tokens += chunkEncodeArr.length;
-    chunks.push(decoder.decode(enc.decode(chunkEncodeArr)));
+      chunks.push(decoder.decode(enc.decode(chunkEncodeArr)));
 
-    startIndex += maxLen - slideLen;
+      startIndex += maxLen - slideLen;
-    endIndex = Math.min(startIndex + maxLen, encodeText.length);
+      endIndex = Math.min(startIndex + maxLen, encodeText.length);
-    chunkEncodeArr = encodeText.slice(Math.min(encodeText.length - slideLen, startIndex), endIndex);
+      chunkEncodeArr = encodeText.slice(
+        Math.min(encodeText.length - slideLen, startIndex),
+        endIndex
+      );
+    }
+
+    return {
+      chunks,
+      tokens
+    };
+  } catch (error) {
+    return Promise.reject(error);
   }
-
-  return {
-    chunks,
-    tokens
-  };
 };
 
 export const fileToBase64 = (file: File) => {

src/utils/plugin/google.ts

@@ -0,0 +1,34 @@
+import axios from 'axios';
+import { Obj2Query } from '../tools';
+
+export const getClientToken = (googleVerKey: string) => {
+  if (typeof grecaptcha === 'undefined' || !grecaptcha?.ready) return '';
+  return new Promise<string>((resolve, reject) => {
+    grecaptcha.ready(async () => {
+      try {
+        const token = await grecaptcha.execute(googleVerKey, {
+          action: 'submit'
+        });
+        resolve(token);
+      } catch (error) {
+        reject(error);
+      }
+    });
+  });
+};
+
+// service run
+export const authGoogleToken = async (data: {
+  secret: string;
+  response: string;
+  remoteip?: string;
+}) => {
+  const res = await axios.post<{ score?: number; success: boolean; 'error-codes': string[] }>(
+    `https://www.recaptcha.net/recaptcha/api/siteverify?${Obj2Query(data)}`
+  );
+
+  if (res.data.success && res.data.score && res.data.score >= 0.7) {
+    return Promise.resolve('');
+  }
+  return Promise.reject(res.data['error-codes'][0] || '非法环境');
+};