perf: export data

docs
feat: insert data de-weight;perf: input queue
2023-05-28 21:13:17 +08:00 · 2023-05-28 21:02:18 +08:00 · 2023-05-28 20:13:19 +08:00 · 2023-05-28 16:16:59 +08:00 · 2023-05-28 10:23:14 +08:00 · 2023-05-27 15:10:19 +08:00
63 changed files with 1005 additions and 824 deletions
--- a/.env.template
+++ b/.env.template
@@ -1,9 +1,6 @@
 # proxy
 # AXIOS_PROXY_HOST=127.0.0.1
 # AXIOS_PROXY_PORT=7890
-# 是否开启队列任务。 1-开启，0-关闭（请求parentUrl去执行任务,单机时直接填1）
-queueTask=1
-parentUrl=https://hostname/api/openapi/startEvents
 # email
 MY_MAIL=xxx@qq.com
 MAILE_CODE=xxx
@@ -21,7 +18,8 @@ SENSITIVE_CHECK=1
 # openai
 # OPENAI_BASE_URL=https://api.openai.com/v1
 # OPENAI_BASE_URL_AUTH=可选的安全凭证(不需要的时候，记得去掉)
-OPENAIKEY=sk-xxx
+OPENAIKEY=sk-xxx # 对话用的key
+OPENAI_TRAINING_KEY=sk-xxx # 训练用的key
 GPT4KEY=sk-xxx
 # claude
 CLAUDE_BASE_URL=calude模型请求地址
--- a/docs/deploy/fastgpt/docker-compose.yml
+++ b/docs/deploy/fastgpt/docker-compose.yml
@@ -17,7 +17,7 @@ services:
      - /root/fastgpt/pg/data:/var/lib/postgresql/data
      - /etc/localtime:/etc/localtime:ro
  mongodb:
-    image: mongo:6.0.4
+    image: mongo:5.0.18
    container_name: mongo
    restart: always
    ports:
@@ -35,13 +35,10 @@ services:
    network_mode: host
    restart: always
    container_name: fastgpt
-    environment:
+    environment: # 可选的变量，不需要的话需要去掉
      # proxy（可选）
      - AXIOS_PROXY_HOST=127.0.0.1
      - AXIOS_PROXY_PORT=7890
-      # 是否开启队列任务。 1-开启，0-关闭（请求 parentUrl 去执行任务,单机时直接填1）
-      - queueTask=1
-      - parentUrl=https://hostname/api/openapi/startEvents
      # 发送邮箱验证码配置。用的是QQ邮箱。参考 nodeMail 获取MAILE_CODE，自行百度。
      - MY_MAIL=xxxx@qq.com
      - MAILE_CODE=xxxx
@@ -50,11 +47,17 @@ services:
      - aliAccessKeySecret=xxxx
      - aliSignName=xxxxx
      - aliTemplateCode=SMS_xxxx
+      # google V3 安全校验（可选）
+      - CLIENT_GOOGLE_VER_TOKEN=xxx
+      - SERVICE_GOOGLE_VER_TOKEN=xx
+      # QA和向量生成最大进程数
+      - QA_MAX_PROCESS=10
+      - VECTOR_MAX_PROCESS=10
      # token加密凭证（随便填，作为登录凭证）
      - TOKEN_KEY=xxxx
      # root key, 最高权限，可以内部接口互相调用
      - ROOT_KEY=xxx
-      # 是否进行安全校验(1: 开启，0: 关闭)
+      # 是否进行内容安全校验(1: 开启，0: 关闭)
      - SENSITIVE_CHECK=1
      # 和上方mongo镜像的username,password对应
      - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin
@@ -66,7 +69,8 @@ services:
      - PG_PASSWORD=1234 # POSTGRES_PASSWORD
      - PG_DB_NAME=fastgpt # POSTGRES_DB
      # openai
-      - OPENAIKEY=sk-xxxxx
+      - OPENAIKEY=sk-xxxxx,sk-xxx # 对话用的key，多个key，逗号分开
+      - OPENAI_TRAINING_KEY=sk-xxx,sk-xxxx # 训练用的key
      - GPT4KEY=sk-xxx
      - OPENAI_BASE_URL=https://api.openai.com/v1
      - OPENAI_BASE_URL_AUTH=可选的安全凭证
--- a/docs/deploy/fastgpt/pg/init.sql
+++ b/docs/deploy/fastgpt/pg/init.sql
@@ -1,21 +1,20 @@
 set -e
 psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL

-CREATE EXTENSION vector;
+CREATE EXTENSION IF NOT EXISTS vector;
 -- init table
-CREATE TABLE modelData (
+CREATE TABLE IF NOT EXISTS modelData (
    id BIGSERIAL PRIMARY KEY,
-    vector VECTOR(1536),
-    status VARCHAR(50) NOT NULL,
+    vector VECTOR(1536) NOT NULL,
    user_id VARCHAR(50) NOT NULL,
-    model_id VARCHAR(50),
-    kb_id VARCHAR(50),
+    kb_id VARCHAR(50) NOT NULL,
    q TEXT NOT NULL,
    a TEXT NOT NULL
 );
-- create index
-CREATE INDEX modelData_status_index ON modelData USING HASH (status);
-CREATE INDEX modelData_userId_index ON modelData USING HASH (user_id);
-CREATE INDEX modelData_userId_index ON modelData USING HASH (model_id);
-CREATE INDEX modelData_kbId_index ON modelData USING HASH (kb_id);
-EOSQL
+-- 索引设置，按需取
+-- CREATE INDEX IF NOT EXISTS modelData_userId_index ON modelData USING HASH (user_id);
+-- CREATE INDEX IF NOT EXISTS modelData_kbId_index ON modelData USING HASH (kb_id);
+-- CREATE INDEX IF NOT EXISTS idx_model_data_md5_q_a_user_id_kb_id ON modelData (md5(q), md5(a), user_id, kb_id);
+-- CREATE INDEX IF NOT EXISTS vector_index ON modelData USING ivfflat (vector vector_cosine_ops) WITH (lists = 1000);
+-- vector 索引，可以到 pg vector 去配置，根据数据量去配置
+EOSQL
--- a/docs/deploy/mac.md
+++ b/docs/deploy/mac.md
@@ -36,7 +36,6 @@ mongo pg
 AXIOS_PROXY_HOST=127.0.0.1
 AXIOS_PROXY_PORT_FAST=7890
 AXIOS_PROXY_PORT_NORMAL=7890
-queueTask=1
 # email
 MY_MAIL= {Your Mail}
 MAILE_CODE={Yoir Mail code}
@@ -48,7 +47,8 @@ aliTemplateCode=SMS_xxx
 # token
 TOKEN_KEY=sswada
 # openai
-OPENAIKEY={Your openapi key}
+OPENAIKEY=sk-xxx # 对话用的key
+OPENAI_TRAINING_KEY=sk-xxx # 训练用的key
 # db
 MONGODB_URI=mongodb://username:password@0.0.0.0:27017/test?authSource=admin
 PG_HOST=0.0.0.0
--- a/docs/dev/README.md
+++ b/docs/dev/README.md
@@ -2,7 +2,7 @@

 第一次开发，请先[部署教程](../deploy/docker.md)，需要部署数据库.

-## 环境变量配置
+## 环境变量配置 (可能更新不及时，以 docker-compose 里的变量为准)

 复制.env.template 文件，生成一个.env.local 环境变量文件夹，修改.env.local 里内容。

@@ -10,9 +10,6 @@
 # proxy（可选）
 AXIOS_PROXY_HOST=127.0.0.1
 AXIOS_PROXY_PORT=7890
-# 是否开启队列任务。 1-开启，0-关闭（请求parentUrl去执行任务,单机时直接填1）
-queueTask=1
-parentUrl=https://hostname/api/openapi/startEvents
 # email
 MY_MAIL=xxx@qq.com
 MAILE_CODE=xxx
@@ -30,7 +27,8 @@ SENSITIVE_CHECK=1
 # openai
 # OPENAI_BASE_URL=https://api.openai.com/v1
 # OPENAI_BASE_URL_AUTH=可选的安全凭证(不需要的时候，记得去掉)
-OPENAIKEY=sk-xxx
+OPENAIKEY=sk-xxx # 对话用的key
+OPENAI_TRAINING_KEY=sk-xxx # 训练用的key
 GPT4KEY=sk-xxx
 # claude
 CLAUDE_BASE_URL=calude模型请求地址
--- a/package.json
+++ b/package.json
@@ -50,10 +50,10 @@
    "react-hook-form": "^7.43.1",
    "react-markdown": "^8.0.5",
    "react-syntax-highlighter": "^15.5.0",
-    "redis": "^4.6.5",
    "rehype-katex": "^6.0.2",
    "remark-gfm": "^3.0.1",
    "remark-math": "^5.1.1",
+    "request-ip": "^3.3.0",
    "sass": "^1.58.3",
    "sharp": "^0.31.3",
    "tunnel": "^0.0.6",
@@ -73,6 +73,7 @@
    "@types/react": "18.0.28",
    "@types/react-dom": "18.0.11",
    "@types/react-syntax-highlighter": "^15.5.6",
+    "@types/request-ip": "^0.0.37",
    "@types/tunnel": "^0.0.3",
    "eslint": "8.34.0",
    "eslint-config-next": "13.1.6",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -25,6 +25,7 @@ specifiers:
  '@types/react': 18.0.28
  '@types/react-dom': 18.0.11
  '@types/react-syntax-highlighter': ^15.5.6
+  '@types/request-ip': ^0.0.37
  '@types/tunnel': ^0.0.3
  axios: ^1.3.3
  cookie: ^0.5.0
@@ -58,10 +59,10 @@ specifiers:
  react-hook-form: ^7.43.1
  react-markdown: ^8.0.5
  react-syntax-highlighter: ^15.5.0
-  redis: ^4.6.5
  rehype-katex: ^6.0.2
  remark-gfm: ^3.0.1
  remark-math: ^5.1.1
+  request-ip: ^3.3.0
  sass: ^1.58.3
  sharp: ^0.31.3
  tunnel: ^0.0.6
@@ -109,10 +110,10 @@ dependencies:
  react-hook-form: registry.npmmirror.com/react-hook-form/7.43.1_react@18.2.0
  react-markdown: registry.npmmirror.com/react-markdown/8.0.5_pmekkgnqduwlme35zpnqhenc34
  react-syntax-highlighter: registry.npmmirror.com/react-syntax-highlighter/15.5.0_react@18.2.0
-  redis: registry.npmmirror.com/redis/4.6.5
  rehype-katex: registry.npmmirror.com/rehype-katex/6.0.2
  remark-gfm: registry.npmmirror.com/remark-gfm/3.0.1
  remark-math: registry.npmmirror.com/remark-math/5.1.1
+  request-ip: 3.3.0
  sass: registry.npmmirror.com/sass/1.58.3
  sharp: registry.npmmirror.com/sharp/0.31.3
  tunnel: registry.npmmirror.com/tunnel/0.0.6
@@ -132,6 +133,7 @@ devDependencies:
  '@types/react': registry.npmmirror.com/@types/react/18.0.28
  '@types/react-dom': registry.npmmirror.com/@types/react-dom/18.0.11
  '@types/react-syntax-highlighter': registry.npmmirror.com/@types/react-syntax-highlighter/15.5.6
+  '@types/request-ip': 0.0.37
  '@types/tunnel': registry.npmmirror.com/@types/tunnel/0.0.3
  eslint: registry.npmmirror.com/eslint/8.34.0
  eslint-config-next: registry.npmmirror.com/eslint-config-next/13.1.6_7kw3g6rralp5ps6mg3uyzz6azm
@@ -302,6 +304,19 @@ packages:
      '@types/unist': 2.0.6
    dev: false

+  /@types/node/14.18.42:
+    resolution: {integrity: sha512-xefu+RBie4xWlK8hwAzGh3npDz/4VhF6icY/shU+zv/1fNn+ZVG7T7CRwe9LId9sAYRPxI+59QBPuKL3WpyGRg==}
+    dev: false
+
+  /@types/node/18.14.0:
+    resolution: {integrity: sha512-5EWrvLmglK+imbCJY0+INViFWUHg1AHel1sq4ZVSfdcNqGy9Edv3UB9IIzzg+xPaUcAgZYcfVs2fBcwDeZzU0A==}
+
+  /@types/request-ip/0.0.37:
+    resolution: {integrity: sha512-uw6/i3rQnpznxD7LtLaeuZytLhKZK6bRoTS6XVJlwxIOoOpEBU7bgKoVXDNtOg4Xl6riUKHa9bjMVrL6ESqYlQ==}
+    dependencies:
+      '@types/node': 18.14.0
+    dev: true
+
  /@types/unist/2.0.6:
    resolution: {integrity: sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==}
    dev: false
@@ -347,6 +362,10 @@ packages:
    engines: {node: '>=0.10.0'}
    dev: false

+  /request-ip/3.3.0:
+    resolution: {integrity: sha512-cA6Xh6e0fDBBBwH77SLJaJPBmD3nWVAcF9/XAcsrIHdjhFzFiB5aNQFytdjCGPezU3ROwrR11IddKAM08vohxA==}
+    dev: false
+
  /saslprep/1.0.3:
    resolution: {integrity: sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==}
    engines: {node: '>=6'}
@@ -4897,72 +4916,6 @@ packages:
    version: 2.11.6
    dev: false

-  registry.npmmirror.com/@redis/bloom/1.2.0_@redis+client@1.5.6:
-    resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/bloom/-/bloom-1.2.0.tgz}
-    id: registry.npmmirror.com/@redis/bloom/1.2.0
-    name: '@redis/bloom'
-    version: 1.2.0
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
-  registry.npmmirror.com/@redis/client/1.5.6:
-    resolution: {integrity: sha512-dFD1S6je+A47Lj22jN/upVU2fj4huR7S9APd7/ziUXsIXDL+11GPYti4Suv5y8FuXaN+0ZG4JF+y1houEJ7ToA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/client/-/client-1.5.6.tgz}
-    name: '@redis/client'
-    version: 1.5.6
-    engines: {node: '>=14'}
-    dependencies:
-      cluster-key-slot: registry.npmmirror.com/cluster-key-slot/1.1.2
-      generic-pool: registry.npmmirror.com/generic-pool/3.9.0
-      yallist: registry.npmmirror.com/yallist/4.0.0
-    dev: false
-
-  registry.npmmirror.com/@redis/graph/1.1.0_@redis+client@1.5.6:
-    resolution: {integrity: sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/graph/-/graph-1.1.0.tgz}
-    id: registry.npmmirror.com/@redis/graph/1.1.0
-    name: '@redis/graph'
-    version: 1.1.0
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
-  registry.npmmirror.com/@redis/json/1.0.4_@redis+client@1.5.6:
-    resolution: {integrity: sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/json/-/json-1.0.4.tgz}
-    id: registry.npmmirror.com/@redis/json/1.0.4
-    name: '@redis/json'
-    version: 1.0.4
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
-  registry.npmmirror.com/@redis/search/1.1.2_@redis+client@1.5.6:
-    resolution: {integrity: sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/search/-/search-1.1.2.tgz}
-    id: registry.npmmirror.com/@redis/search/1.1.2
-    name: '@redis/search'
-    version: 1.1.2
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
-  registry.npmmirror.com/@redis/time-series/1.0.4_@redis+client@1.5.6:
-    resolution: {integrity: sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@redis/time-series/-/time-series-1.0.4.tgz}
-    id: registry.npmmirror.com/@redis/time-series/1.0.4
-    name: '@redis/time-series'
-    version: 1.0.4
-    peerDependencies:
-      '@redis/client': ^1.0.0
-    dependencies:
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-    dev: false
-
  registry.npmmirror.com/@rushstack/eslint-patch/1.2.0:
    resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@rushstack/eslint-patch/-/eslint-patch-1.2.0.tgz}
    name: '@rushstack/eslint-patch'
@@ -5287,16 +5240,11 @@ packages:
    version: 12.20.55
    dev: false

-  registry.npmmirror.com/@types/node/14.18.42:
-    resolution: {integrity: sha512-xefu+RBie4xWlK8hwAzGh3npDz/4VhF6icY/shU+zv/1fNn+ZVG7T7CRwe9LId9sAYRPxI+59QBPuKL3WpyGRg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node/-/node-14.18.42.tgz}
-    name: '@types/node'
-    version: 14.18.42
-    dev: false
-
  registry.npmmirror.com/@types/node/18.14.0:
    resolution: {integrity: sha512-5EWrvLmglK+imbCJY0+INViFWUHg1AHel1sq4ZVSfdcNqGy9Edv3UB9IIzzg+xPaUcAgZYcfVs2fBcwDeZzU0A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node/-/node-18.14.0.tgz}
    name: '@types/node'
    version: 18.14.0
+    dev: true

  registry.npmmirror.com/@types/nodemailer/6.4.7:
    resolution: {integrity: sha512-f5qCBGAn/f0qtRcd4SEn88c8Fp3Swct1731X4ryPKqS61/A3LmmzN8zaEz7hneJvpjFbUUgY7lru/B/7ODTazg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/nodemailer/-/nodemailer-6.4.7.tgz}
@@ -5395,7 +5343,7 @@ packages:
    name: '@types/whatwg-url'
    version: 8.2.2
    dependencies:
-      '@types/node': registry.npmmirror.com/@types/node/18.14.0
+      '@types/node': 18.14.0
      '@types/webidl-conversions': registry.npmmirror.com/@types/webidl-conversions/7.0.0
    dev: false

@@ -5404,7 +5352,7 @@ packages:
    name: '@types/xml2js'
    version: 0.4.11
    dependencies:
-      '@types/node': registry.npmmirror.com/@types/node/18.14.0
+      '@types/node': 18.14.0
    dev: false

  registry.npmmirror.com/@typescript-eslint/parser/5.52.0_7kw3g6rralp5ps6mg3uyzz6azm:
@@ -6117,13 +6065,6 @@ packages:
    version: 0.0.1
    dev: false

-  registry.npmmirror.com/cluster-key-slot/1.1.2:
-    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz}
-    name: cluster-key-slot
-    version: 1.1.2
-    engines: {node: '>=0.10.0'}
-    dev: false
-
  registry.npmmirror.com/color-convert/1.9.3:
    resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/color-convert/-/color-convert-1.9.3.tgz}
    name: color-convert
@@ -7514,13 +7455,6 @@ packages:
    version: 1.2.3
    dev: true

-  registry.npmmirror.com/generic-pool/3.9.0:
-    resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/generic-pool/-/generic-pool-3.9.0.tgz}
-    name: generic-pool
-    version: 3.9.0
-    engines: {node: '>= 4'}
-    dev: false
-
  registry.npmmirror.com/gensync/1.0.0-beta.2:
    resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/gensync/-/gensync-1.0.0-beta.2.tgz}
    name: gensync
@@ -7926,7 +7860,7 @@ packages:
    name: httpx
    version: 2.2.7
    dependencies:
-      '@types/node': registry.npmmirror.com/@types/node/14.18.42
+      '@types/node': 14.18.42
      debug: registry.npmmirror.com/debug/4.3.4
    transitivePeerDependencies:
      - supports-color
@@ -10515,19 +10449,6 @@ packages:
      picomatch: registry.npmmirror.com/picomatch/2.3.1
    dev: false

-  registry.npmmirror.com/redis/4.6.5:
-    resolution: {integrity: sha512-O0OWA36gDQbswOdUuAhRL6mTZpHFN525HlgZgDaVNgCJIAZR3ya06NTESb0R+TUZ+BFaDpz6NnnVvoMx9meUFg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/redis/-/redis-4.6.5.tgz}
-    name: redis
-    version: 4.6.5
-    dependencies:
-      '@redis/bloom': registry.npmmirror.com/@redis/bloom/1.2.0_@redis+client@1.5.6
-      '@redis/client': registry.npmmirror.com/@redis/client/1.5.6
-      '@redis/graph': registry.npmmirror.com/@redis/graph/1.1.0_@redis+client@1.5.6
-      '@redis/json': registry.npmmirror.com/@redis/json/1.0.4_@redis+client@1.5.6
-      '@redis/search': registry.npmmirror.com/@redis/search/1.1.2_@redis+client@1.5.6
-      '@redis/time-series': registry.npmmirror.com/@redis/time-series/1.0.4_@redis+client@1.5.6
-    dev: false
-
  registry.npmmirror.com/reflect-metadata/0.1.13:
    resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz}
    name: reflect-metadata
--- a/src/api/plugins/kb.ts
+++ b/src/api/plugins/kb.ts
@@ -1,8 +1,11 @@
 import { GET, POST, PUT, DELETE } from '../request';
 import type { KbItemType } from '@/types/plugin';
 import { RequestPaging } from '@/types/index';
-import { SplitTextTypEnum } from '@/constants/plugin';
-import { KbDataItemType } from '@/types/plugin';
+import { TrainingModeEnum } from '@/constants/plugin';
+import {
+  Props as PushDataProps,
+  Response as PushDateResponse
+} from '@/pages/api/openapi/kb/pushData';

 export type KbUpdateParams = { id: string; name: string; tags: string; avatar: string };

@@ -29,16 +32,22 @@ export const getKbDataList = (data: GetKbDataListProps) =>
 * 获取导出数据（不分页）
 */
 export const getExportDataList = (kbId: string) =>
-  GET<[string, string][]>(`/plugins/kb/data/exportModelData?kbId=${kbId}`);
+  GET<[string, string][]>(
+    `/plugins/kb/data/exportModelData`,
+    { kbId },
+    {
+      timeout: 600000
+    }
+  );

 /**
 * 获取模型正在拆分数据的数量
 */
-export const getTrainingData = (kbId: string) =>
-  GET<{
-    splitDataQueue: number;
-    embeddingQueue: number;
-  }>(`/plugins/kb/data/getTrainingData?kbId=${kbId}`);
+export const getTrainingData = (data: { kbId: string; init: boolean }) =>
+  POST<{
+    qaListLen: number;
+    vectorListLen: number;
+  }>(`/plugins/kb/data/getTrainingData`, data);

 export const getKbDataItemById = (dataId: string) =>
  GET(`/plugins/kb/data/getDataById`, { dataId });
@@ -46,10 +55,8 @@ export const getKbDataItemById = (dataId: string) =>
 /**
 * 直接push数据
 */
-export const postKbDataFromList = (data: {
-  kbId: string;
-  data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
-}) => POST(`/openapi/kb/pushData`, data);
+export const postKbDataFromList = (data: PushDataProps) =>
+  POST<PushDateResponse>(`/openapi/kb/pushData`, data);

 /**
 * 更新一条数据
@@ -69,5 +76,5 @@ export const postSplitData = (data: {
  kbId: string;
  chunks: string[];
  prompt: string;
-  mode: `${SplitTextTypEnum}`;
-}) => POST(`/openapi/text/splitText`, data);
+  mode: `${TrainingModeEnum}`;
+}) => POST(`/openapi/text/pushData`, data);
--- a/src/api/request.ts
+++ b/src/api/request.ts
@@ -5,6 +5,7 @@ import { TOKEN_ERROR_CODE } from '@/service/errorCode';
 interface ConfigType {
  headers?: { [key: string]: string };
  hold?: boolean;
+  timeout?: number;
 }
 interface ResponseDataType {
  code: number;
--- a/src/api/system.ts
+++ b/src/api/system.ts
@@ -1,6 +1,7 @@
 import { GET, POST, PUT } from './request';
 import type { ChatModelItemType } from '@/constants/model';
+import type { InitDateResponse } from '@/pages/api/system/getInitData';

-export const getFilling = () => GET<{ beianText: string }>('/system/getFiling');
+export const getInitData = () => GET<InitDateResponse>('/system/getInitData');

 export const getSystemModelList = () => GET<ChatModelItemType[]>('/system/getModels');
--- a/src/api/user.ts
+++ b/src/api/user.ts
@@ -6,13 +6,11 @@ import { UserBillType, UserType, UserUpdateParams } from '@/types/user';
 import type { PagingData, RequestPaging } from '@/types';
 import { PaySchema } from '@/types/mongoSchema';

-export const sendAuthCode = ({
-  username,
-  type
-}: {
+export const sendAuthCode = (data: {
  username: string;
  type: `${UserAuthTypeEnum}`;
-}) => GET('/user/sendAuthCode', { username, type });
+  googleToken: string;
+}) => POST('/user/sendAuthCode', data);

 export const getTokenLogin = () => GET<UserType>('/user/tokenLogin');

--- a/src/components/Avatar/index.tsx
+++ b/src/components/Avatar/index.tsx
@@ -7,7 +7,7 @@ const Avatar = ({ w = '30px', ...props }: ImageProps) => {
    <Image
      fallbackSrc="/icon/logo.png"
      borderRadius={'50%'}
-      objectFit={'contain'}
+      objectFit={'cover'}
      alt=""
      w={w}
      h={w}
--- a/src/constants/model.ts
+++ b/src/constants/model.ts
@@ -30,7 +30,7 @@ export const ChatModelMap = {
    chatModel: OpenAiChatEnum.GPT35,
    name: 'ChatGpt',
    contextMaxToken: 4096,
-    systemMaxToken: 2400,
+    systemMaxToken: 2700,
    maxTemperature: 1.2,
    price: 2.5
  },
@@ -38,7 +38,7 @@ export const ChatModelMap = {
    chatModel: OpenAiChatEnum.GPT4,
    name: 'Gpt4',
    contextMaxToken: 8000,
-    systemMaxToken: 3000,
+    systemMaxToken: 4000,
    maxTemperature: 1.2,
    price: 50
  },
@@ -46,7 +46,7 @@ export const ChatModelMap = {
    chatModel: OpenAiChatEnum.GPT432k,
    name: 'Gpt4-32k',
    contextMaxToken: 32000,
-    systemMaxToken: 3000,
+    systemMaxToken: 8000,
    maxTemperature: 1.2,
    price: 90
  },
@@ -54,7 +54,7 @@ export const ChatModelMap = {
    chatModel: ClaudeEnum.Claude,
    name: 'Claude(免费体验)',
    contextMaxToken: 9000,
-    systemMaxToken: 2400,
+    systemMaxToken: 2700,
    maxTemperature: 1,
    price: 0
  }
@@ -96,39 +96,29 @@ export const formatModelStatus = {
  }
 };

-export enum ModelDataStatusEnum {
-  ready = 'ready',
-  waiting = 'waiting'
-}
-
-export const ModelDataStatusMap: Record<`${ModelDataStatusEnum}`, string> = {
-  ready: '训练完成',
-  waiting: '训练中'
-};
-
 /* 知识库搜索时的配置 */
 // 搜索方式
-export enum ModelVectorSearchModeEnum {
+export enum appVectorSearchModeEnum {
  hightSimilarity = 'hightSimilarity', // 高相似度+禁止回复
  lowSimilarity = 'lowSimilarity', // 低相似度
  noContext = 'noContex' // 高相似度+无上下文回复
 }
 export const ModelVectorSearchModeMap: Record<
-  `${ModelVectorSearchModeEnum}`,
+  `${appVectorSearchModeEnum}`,
  {
    text: string;
    similarity: number;
  }
 > = {
-  [ModelVectorSearchModeEnum.hightSimilarity]: {
+  [appVectorSearchModeEnum.hightSimilarity]: {
    text: '高相似度, 无匹配时拒绝回复',
    similarity: 0.18
  },
-  [ModelVectorSearchModeEnum.noContext]: {
+  [appVectorSearchModeEnum.noContext]: {
    text: '高相似度，无匹配时直接回复',
    similarity: 0.18
  },
-  [ModelVectorSearchModeEnum.lowSimilarity]: {
+  [appVectorSearchModeEnum.lowSimilarity]: {
    text: '低相似度匹配',
    similarity: 0.7
  }
@@ -143,7 +133,7 @@ export const defaultModel: ModelSchema = {
  updateTime: Date.now(),
  chat: {
    relatedKbs: [],
-    searchMode: ModelVectorSearchModeEnum.hightSimilarity,
+    searchMode: appVectorSearchModeEnum.hightSimilarity,
    systemPrompt: '',
    temperature: 0,
    chatModel: OpenAiChatEnum.GPT35
--- a/src/constants/plugin.ts
+++ b/src/constants/plugin.ts
@@ -1,14 +1,8 @@
-export enum SplitTextTypEnum {
+export enum TrainingModeEnum {
  'qa' = 'qa',
-  'subsection' = 'subsection'
-}
-
-export enum PluginTypeEnum {
-  LLM = 'LLM',
-  Text = 'Text',
-  Function = 'Function'
-}
-
-export enum PluginParamsTypeEnum {
-  'Text' = 'text'
+  'index' = 'index'
 }
+export const TrainingTypeMap = {
+  [TrainingModeEnum.qa]: 'qa',
+  [TrainingModeEnum.index]: 'index'
+};
--- a/src/hooks/useSendCode.ts
+++ b/src/hooks/useSendCode.ts
@@ -3,8 +3,13 @@ import { sendAuthCode } from '@/api/user';
 import { UserAuthTypeEnum } from '@/constants/common';
 let timer: any;
 import { useToast } from './useToast';
+import { getClientToken } from '@/utils/plugin/google';
+import { useGlobalStore } from '@/store/global';

 export const useSendCode = () => {
+  const {
+    initData: { googleVerKey }
+  } = useGlobalStore();
  const { toast } = useToast();
  const [codeSending, setCodeSending] = useState(false);
  const [codeCountDown, setCodeCountDown] = useState(0);
@@ -24,7 +29,8 @@ export const useSendCode = () => {
      try {
        await sendAuthCode({
          username,
-          type
+          type,
+          googleToken: await getClientToken(googleVerKey)
        });
        setCodeCountDown(60);
        timer = setInterval(() => {
@@ -48,7 +54,7 @@ export const useSendCode = () => {
      }
      setCodeSending(false);
    },
-    [toast]
+    [googleVerKey, toast]
  );

  return {
--- a/src/pages/_app.tsx
+++ b/src/pages/_app.tsx
@@ -10,7 +10,7 @@ import NProgress from 'nprogress'; //nprogress module
 import Router from 'next/router';
 import 'nprogress/nprogress.css';
 import '../styles/reset.scss';
-import { useToast } from '@/hooks/useToast';
+import { useGlobalStore } from '@/store/global';

 //Binding events.
 Router.events.on('routeChangeStart', () => NProgress.start());
@@ -29,23 +29,20 @@ const queryClient = new QueryClient({
 });

 export default function App({ Component, pageProps }: AppProps) {
-  const { toast } = useToast();
-  // 校验是否支持 click 事件
+  const {
+    loadInitData,
+    initData: { googleVerKey }
+  } = useGlobalStore();
+
  useEffect(() => {
-    if (typeof document.createElement('div').click !== 'function') {
-      toast({
-        title: '你的浏览器版本过低',
-        status: 'warning'
-      });
-    }
-  }, [toast]);
+    loadInitData();
+  }, []);

  return (
    <>
      <Head>
        <title>Fast GPT</title>
        <meta name="description" content="Generated by Fast GPT" />
-
        <meta
          name="viewport"
          content="width=device-width,initial-scale=1.0,maximum-scale=1.0,minimum-scale=1.0,user-scalable=no, viewport-fit=cover"
@@ -55,6 +52,12 @@ export default function App({ Component, pageProps }: AppProps) {
      <Script src="/js/qrcode.min.js" strategy="lazyOnload"></Script>
      <Script src="/js/pdf.js" strategy="lazyOnload"></Script>
      <Script src="/js/html2pdf.bundle.min.js" strategy="lazyOnload"></Script>
+      {googleVerKey && (
+        <Script
+          src={`https://www.recaptcha.net/recaptcha/api.js?render=${googleVerKey}`}
+          strategy="lazyOnload"
+        ></Script>
+      )}
      <Script src="/js/particles.js"></Script>
      <QueryClientProvider client={queryClient}>
        <ChakraProvider theme={theme}>
--- a/src/pages/api/admin/countTraining.ts
+++ b/src/pages/api/admin/countTraining.ts
@@ -0,0 +1,37 @@
+// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { authUser } from '@/service/utils/auth';
+import { connectToDatabase, TrainingData } from '@/service/mongo';
+import { TrainingModeEnum } from '@/constants/plugin';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    await authUser({ req, authRoot: true });
+
+    await connectToDatabase();
+
+    // split queue data
+    const result = await TrainingData.aggregate([
+      {
+        $group: {
+          _id: '$mode',
+          count: { $sum: 1 }
+        }
+      }
+    ]);
+
+    jsonRes(res, {
+      data: {
+        qaListLen: result.find((item) => item._id === TrainingModeEnum.qa)?.count || 0,
+        vectorListLen: result.find((item) => item._id === TrainingModeEnum.index)?.count || 0
+      }
+    });
+  } catch (error) {
+    console.log(error);
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
--- a/src/pages/api/chat/saveChat.ts
+++ b/src/pages/api/chat/saveChat.ts
@@ -58,7 +58,11 @@ export async function saveChat({
    obj: item.obj,
    value: item.value,
    systemPrompt: item.systemPrompt,
-    quote: item.quote || []
+    quote:
+      item.quote?.map((item) => ({
+        ...item,
+        isEdit: false
+      })) || []
  }));

  // 没有 chatId, 创建一个对话
--- a/src/pages/api/openapi/kb/appKbSearch.ts
+++ b/src/pages/api/openapi/kb/appKbSearch.ts
@@ -5,12 +5,11 @@ import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
 import type { ChatItemSimpleType } from '@/types/chat';
 import type { ModelSchema } from '@/types/mongoSchema';
-import { ModelVectorSearchModeEnum } from '@/constants/model';
+import { appVectorSearchModeEnum } from '@/constants/model';
 import { authModel } from '@/service/utils/auth';
 import { ChatModelMap } from '@/constants/model';
 import { ChatRoleEnum } from '@/constants/chat';
 import { openaiEmbedding } from '../plugin/openaiEmbedding';
-import { ModelDataStatusEnum } from '@/constants/model';
 import { modelToolMap } from '@/utils/plugin';

 export type QuoteItemType = { id: string; q: string; a: string; isEdit: boolean };
@@ -92,7 +91,8 @@ export async function appKbSearch({
  // get vector
  const promptVectors = await openaiEmbedding({
    userId,
-    input
+    input,
+    type: 'chat'
  });

  // search kb
@@ -101,8 +101,6 @@ export async function appKbSearch({
      PgClient.select<QuoteItemType>('modelData', {
        fields: ['id', 'q', 'a'],
        where: [
-          ['status', ModelDataStatusEnum.ready],
-          'AND',
          `kb_id IN (${model.chat.relatedKbs.map((item) => `'${item}'`).join(',')})`,
          'AND',
          `vector <=> '[${promptVector}]' < ${similarity}`
@@ -138,7 +136,7 @@ export async function appKbSearch({
        obj: ChatRoleEnum.System,
        value: model.chat.systemPrompt
      }
-    : model.chat.searchMode === ModelVectorSearchModeEnum.noContext
+    : model.chat.searchMode === appVectorSearchModeEnum.noContext
    ? {
        obj: ChatRoleEnum.System,
        value: `知识库是关于"${model.name}"的内容,根据知识库内容回答问题.`
@@ -176,7 +174,7 @@ export async function appKbSearch({
  const systemPrompt = sliceResult.flat().join('\n').trim();

  /* 高相似度+不回复 */
-  if (!systemPrompt && model.chat.searchMode === ModelVectorSearchModeEnum.hightSimilarity) {
+  if (!systemPrompt && model.chat.searchMode === appVectorSearchModeEnum.hightSimilarity) {
    return {
      code: 201,
      rawSearch: [],
@@ -190,7 +188,7 @@ export async function appKbSearch({
    };
  }
  /* 高相似度+无上下文，不添加额外知识,仅用系统提示词 */
-  if (!systemPrompt && model.chat.searchMode === ModelVectorSearchModeEnum.noContext) {
+  if (!systemPrompt && model.chat.searchMode === appVectorSearchModeEnum.noContext) {
    return {
      code: 200,
      rawSearch: [],
--- a/src/pages/api/openapi/kb/pushData.ts
+++ b/src/pages/api/openapi/kb/pushData.ts
@@ -1,84 +1,45 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import type { KbDataItemType } from '@/types/plugin';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase } from '@/service/mongo';
+import { connectToDatabase, TrainingData } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
-import { generateVector } from '@/service/events/generateVector';
-import { PgClient, insertKbItem } from '@/service/pg';
 import { authKb } from '@/service/utils/auth';
 import { withNextCors } from '@/service/utils/tools';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { startQueue } from '@/service/utils/tools';
+import { PgClient } from '@/service/pg';
+
+export type Props = {
+  kbId: string;
+  data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
+  mode: `${TrainingModeEnum}`;
+  prompt?: string;
+};
+
+export type Response = {
+  insertLen: number;
+};

 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
-    const {
-      kbId,
-      data,
-      formatLineBreak = true
-    } = req.body as {
-      kbId: string;
-      formatLineBreak?: boolean;
-      data: { a: KbDataItemType['a']; q: KbDataItemType['q'] }[];
-    };
+    const { kbId, data, mode, prompt } = req.body as Props;

    if (!kbId || !Array.isArray(data)) {
      throw new Error('缺少参数');
    }
-
    await connectToDatabase();

    // 凭证校验
    const { userId } = await authUser({ req });

-    await authKb({
-      userId,
-      kbId
-    });
-
-    // 过滤重复的内容
-    const searchRes = await Promise.allSettled(
-      data.map(async ({ q, a = '' }) => {
-        if (!q) {
-          return Promise.reject('q为空');
-        }
-
-        if (formatLineBreak) {
-          q = q.replace(/\\n/g, '\n');
-          a = a.replace(/\\n/g, '\n');
-        }
-
-        // Exactly the same data, not push
-        try {
-          const count = await PgClient.count('modelData', {
-            where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['q', q], 'AND', ['a', a]]
-          });
-          if (count > 0) {
-            return Promise.reject('已经存在');
-          }
-        } catch (error) {
-          error;
-        }
-        return Promise.resolve({
-          q,
-          a
-        });
+    jsonRes<Response>(res, {
+      data: await pushDataToKb({
+        kbId,
+        data,
+        userId,
+        mode,
+        prompt
      })
-    );
-    const filterData = searchRes
-      .filter((item) => item.status === 'fulfilled')
-      .map<{ q: string; a: string }>((item: any) => item.value);
-
-    // 插入记录
-    const insertRes = await insertKbItem({
-      userId,
-      kbId,
-      data: filterData
-    });
-
-    generateVector();
-
-    jsonRes(res, {
-      message: `共插入 ${insertRes.rowCount} 条数据`,
-      data: insertRes.rowCount
    });
  } catch (err) {
    jsonRes(res, {
@@ -88,10 +49,100 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
  }
 });

+export async function pushDataToKb({
+  userId,
+  kbId,
+  data,
+  mode,
+  prompt
+}: { userId: string } & Props): Promise<Response> {
+  await authKb({
+    userId,
+    kbId
+  });
+
+  // 过滤重复的 qa 内容
+  const set = new Set();
+  const filterData: {
+    a: string;
+    q: string;
+  }[] = [];
+
+  data.forEach((item) => {
+    const text = item.q + item.a;
+    if (!set.has(text)) {
+      filterData.push(item);
+      set.add(text);
+    }
+  });
+
+  // 数据库去重
+  const insertData = (
+    await Promise.allSettled(
+      filterData.map(async ({ q, a = '' }) => {
+        if (mode !== TrainingModeEnum.index) {
+          return Promise.resolve({
+            q,
+            a
+          });
+        }
+
+        if (!q) {
+          return Promise.reject('q为空');
+        }
+
+        q = q.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
+        a = a.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
+
+        // Exactly the same data, not push
+        try {
+          const { rows } = await PgClient.query(`
+            SELECT COUNT(*) > 0 AS exists
+            FROM  modelData 
+            WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND kb_id='${kbId}'
+          `);
+          const exists = rows[0]?.exists || false;
+
+          if (exists) {
+            return Promise.reject('已经存在');
+          }
+        } catch (error) {
+          console.log(error);
+          error;
+        }
+        return Promise.resolve({
+          q,
+          a
+        });
+      })
+    )
+  )
+    .filter((item) => item.status === 'fulfilled')
+    .map<{ q: string; a: string }>((item: any) => item.value);
+
+  // 插入记录
+  await TrainingData.insertMany(
+    insertData.map((item) => ({
+      q: item.q,
+      a: item.a,
+      userId,
+      kbId,
+      mode,
+      prompt
+    }))
+  );
+
+  insertData.length > 0 && startQueue();
+
+  return {
+    insertLen: insertData.length
+  };
+}
+
 export const config = {
  api: {
    bodyParser: {
-      sizeLimit: '100mb'
+      sizeLimit: '20mb'
    }
  }
 };
--- a/src/pages/api/openapi/kb/updateData.ts
+++ b/src/pages/api/openapi/kb/updateData.ts
@@ -1,14 +1,13 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
-import { ModelDataStatusEnum } from '@/constants/model';
-import { generateVector } from '@/service/events/generateVector';
 import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
+import { openaiEmbedding } from '../plugin/openaiEmbedding';

 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
-    const { dataId, a, q } = req.body as { dataId: string; a: string; q?: string };
+    const { dataId, a = '', q = '' } = req.body as { dataId: string; a?: string; q?: string };

    if (!dataId) {
      throw new Error('缺少参数');
@@ -17,22 +16,32 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
    // 凭证校验
    const { userId } = await authUser({ req });

+    // get vector
+    const vector = await (async () => {
+      if (q) {
+        return openaiEmbedding({
+          userId,
+          input: [q],
+          type: 'chat'
+        });
+      }
+      return [];
+    })();
+
    // 更新 pg 内容.仅修改a，不需要更新向量。
    await PgClient.update('modelData', {
      where: [['id', dataId], 'AND', ['user_id', userId]],
      values: [
-        { key: 'a', value: a },
+        { key: 'a', value: a.replace(/'/g, '"') },
        ...(q
          ? [
-              { key: 'q', value: q },
-              { key: 'status', value: ModelDataStatusEnum.waiting }
+              { key: 'q', value: q.replace(/'/g, '"') },
+              { key: 'vector', value: `[${vector[0]}]` }
            ]
          : [])
      ]
    });

-    q && generateVector();
-
    jsonRes(res);
  } catch (err) {
    jsonRes(res, {
--- a/src/pages/api/openapi/plugin/openaiEmbedding.ts
+++ b/src/pages/api/openapi/plugin/openaiEmbedding.ts
@@ -1,30 +1,31 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { authUser } from '@/service/utils/auth';
-import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
 import { getApiKey } from '@/service/utils/auth';
 import { getOpenAIApi } from '@/service/utils/chat/openai';
 import { embeddingModel } from '@/constants/model';
 import { axiosConfig } from '@/service/utils/tools';
 import { pushGenerateVectorBill } from '@/service/events/pushBill';
+import { ApiKeyType } from '@/service/utils/auth';

 type Props = {
  input: string[];
+  type?: ApiKeyType;
 };
 type Response = number[][];

 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
    const { userId } = await authUser({ req });
-    let { input } = req.query as Props;
+    let { input, type } = req.query as Props;

    if (!Array.isArray(input)) {
      throw new Error('缺少参数');
    }

    jsonRes<Response>(res, {
-      data: await openaiEmbedding({ userId, input, mustPay: true })
+      data: await openaiEmbedding({ userId, input, mustPay: true, type })
    });
  } catch (err) {
    console.log(err);
@@ -38,12 +39,14 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
 export async function openaiEmbedding({
  userId,
  input,
-  mustPay = false
+  mustPay = false,
+  type = 'chat'
 }: { userId: string; mustPay?: boolean } & Props) {
  const { userOpenAiKey, systemAuthKey } = await getApiKey({
    model: 'gpt-3.5-turbo',
    userId,
-    mustPay
+    mustPay,
+    type
  });

  // 获取 chatAPI
--- a/src/pages/api/openapi/startEvents.ts
+++ b/src/pages/api/openapi/startEvents.ts
@@ -1,19 +0,0 @@
-// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { generateQA } from '@/service/events/generateQA';
-import { generateVector } from '@/service/events/generateVector';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    generateQA();
-    generateVector();
-
-    jsonRes(res);
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}
--- a/src/pages/api/openapi/text/gptMessagesSlice.ts
+++ b/src/pages/api/openapi/text/gptMessagesSlice.ts
@@ -62,5 +62,5 @@ export function gpt_chatItemTokenSlice({
    }
  }

-  return result;
+  return result.length === 0 && messages[0] ? [messages[0]] : result;
 }
--- a/src/pages/api/openapi/text/sensitiveCheck.ts
+++ b/src/pages/api/openapi/text/sensitiveCheck.ts
@@ -17,7 +17,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    const { input } = req.body as TextPluginRequestParams;

    const response = await axios({
-      ...axiosConfig(getSystemOpenAiKey()),
+      ...axiosConfig(getSystemOpenAiKey('chat')),
      method: 'POST',
      url: `/moderations`,
      data: {
--- a/src/pages/api/openapi/text/splitText.ts
+++ b/src/pages/api/openapi/text/splitText.ts
@@ -1,73 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, SplitData } from '@/service/mongo';
-import { authKb, authUser } from '@/service/utils/auth';
-import { generateVector } from '@/service/events/generateVector';
-import { generateQA } from '@/service/events/generateQA';
-import { insertKbItem } from '@/service/pg';
-import { SplitTextTypEnum } from '@/constants/plugin';
-import { withNextCors } from '@/service/utils/tools';
-
-/* split text */
-export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    const { chunks, kbId, prompt, mode } = req.body as {
-      kbId: string;
-      chunks: string[];
-      prompt: string;
-      mode: `${SplitTextTypEnum}`;
-    };
-    if (!chunks || !kbId || !prompt) {
-      throw new Error('参数错误');
-    }
-    await connectToDatabase();
-
-    const { userId } = await authUser({ req });
-
-    // 验证是否是该用户的 model
-    await authKb({
-      kbId,
-      userId
-    });
-
-    if (mode === SplitTextTypEnum.qa) {
-      // 批量QA拆分插入数据
-      await SplitData.create({
-        userId,
-        kbId,
-        textList: chunks,
-        prompt
-      });
-
-      generateQA();
-    } else if (mode === SplitTextTypEnum.subsection) {
-      // 待优化，直接调用另一个接口
-      // 插入记录
-      await insertKbItem({
-        userId,
-        kbId,
-        data: chunks.map((item) => ({
-          q: item,
-          a: ''
-        }))
-      });
-
-      generateVector();
-    }
-
-    jsonRes(res);
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-});
-
-export const config = {
-  api: {
-    bodyParser: {
-      sizeLimit: '100mb'
-    }
-  }
-};
--- a/src/pages/api/plugins/kb/data/exportModelData.ts
+++ b/src/pages/api/plugins/kb/data/exportModelData.ts
@@ -69,3 +69,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    });
  }
 }
+
+export const config = {
+  api: {
+    bodyParser: {
+      sizeLimit: '100mb'
+    }
+  }
+};
--- a/src/pages/api/plugins/kb/data/getDataById.ts
+++ b/src/pages/api/plugins/kb/data/getDataById.ts
@@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    const where: any = [['user_id', userId], 'AND', ['id', dataId]];

    const searchRes = await PgClient.select<PgKBDataItemType>('modelData', {
-      fields: ['id', 'q', 'a', 'status'],
+      fields: ['id', 'q', 'a'],
      where,
      limit: 1
    });
--- a/src/pages/api/plugins/kb/data/getDataList.ts
+++ b/src/pages/api/plugins/kb/data/getDataList.ts
@@ -35,7 +35,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    ];

    const searchRes = await PgClient.select<PgKBDataItemType>('modelData', {
-      fields: ['id', 'q', 'a', 'status'],
+      fields: ['id', 'q', 'a'],
      where,
      order: [{ field: 'id', mode: 'DESC' }],
      limit: pageSize,
--- a/src/pages/api/plugins/kb/data/getTrainingData.ts
+++ b/src/pages/api/plugins/kb/data/getTrainingData.ts
@@ -1,14 +1,15 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, SplitData, Model } from '@/service/mongo';
+import { connectToDatabase, TrainingData } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
-import { ModelDataStatusEnum } from '@/constants/model';
-import { PgClient } from '@/service/pg';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { Types } from 'mongoose';
+import { startQueue } from '@/service/utils/tools';

 /* 拆分数据成QA */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
  try {
-    const { kbId } = req.query as { kbId: string };
+    const { kbId, init = false } = req.body as { kbId: string; init: boolean };
    if (!kbId) {
      throw new Error('参数错误');
    }
@@ -17,29 +18,31 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
    const { userId } = await authUser({ req, authToken: true });

    // split queue data
-    const data = await SplitData.find({
-      userId,
-      kbId,
-      textList: { $exists: true, $not: { $size: 0 } }
-    });
-
-    // embedding queue data
-    const embeddingData = await PgClient.count('modelData', {
-      where: [
-        ['user_id', userId],
-        'AND',
-        ['kb_id', kbId],
-        'AND',
-        ['status', ModelDataStatusEnum.waiting]
-      ]
-    });
+    const result = await TrainingData.aggregate([
+      {
+        $match: {
+          userId: new Types.ObjectId(userId),
+          kbId: new Types.ObjectId(kbId)
+        }
+      },
+      {
+        $group: {
+          _id: '$mode',
+          count: { $sum: 1 }
+        }
+      }
+    ]);

    jsonRes(res, {
      data: {
-        splitDataQueue: data.map((item) => item.textList).flat().length,
-        embeddingQueue: embeddingData
+        qaListLen: result.find((item) => item._id === TrainingModeEnum.qa)?.count || 0,
+        vectorListLen: result.find((item) => item._id === TrainingModeEnum.index)?.count || 0
      }
    });
+
+    if (init) {
+      startQueue();
+    }
  } catch (err) {
    jsonRes(res, {
      code: 500,
--- a/src/pages/api/plugins/kb/delete.ts
+++ b/src/pages/api/plugins/kb/delete.ts
@@ -1,8 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, KB } from '@/service/mongo';
+import { connectToDatabase, KB, Model, TrainingData } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { PgClient } from '@/service/pg';
+import { Types } from 'mongoose';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -19,19 +20,30 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<

    await connectToDatabase();

-    // delete mongo data
-    await KB.findOneAndDelete({
-      _id: id,
-      userId
-    });
-
    // delete all pg data
-    // 删除 pg 中所有该模型的数据
    await PgClient.delete('modelData', {
      where: [['user_id', userId], 'AND', ['kb_id', id]]
    });

+    // delete training data
+    await TrainingData.deleteMany({
+      userId,
+      kbId: id
+    });
+
    // delete related model
+    await Model.updateMany(
+      {
+        userId
+      },
+      { $pull: { 'chat.relatedKbs': new Types.ObjectId(id) } }
+    );
+
+    // delete kb data
+    await KB.findOneAndDelete({
+      _id: id,
+      userId
+    });

    jsonRes(res);
  } catch (err) {
--- a/src/pages/api/system/getInitData.ts
+++ b/src/pages/api/system/getInitData.ts
@@ -2,10 +2,16 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';

+export type InitDateResponse = {
+  beianText: string;
+  googleVerKey: string;
+};
+
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  jsonRes(res, {
+  jsonRes<InitDateResponse>(res, {
    data: {
-      beianText: process.env.SAFE_BEIAN_TEXT || ''
+      beianText: process.env.SAFE_BEIAN_TEXT || '',
+      googleVerKey: process.env.CLIENT_GOOGLE_VER_TOKEN || ''
    }
  });
 }
--- a/src/pages/api/user/sendAuthCode.ts
+++ b/src/pages/api/user/sendAuthCode.ts
@@ -7,15 +7,29 @@ import { sendPhoneCode, sendEmailCode } from '@/service/utils/sendNote';
 import { UserAuthTypeEnum } from '@/constants/common';
 import { customAlphabet } from 'nanoid';
 const nanoid = customAlphabet('123456789', 6);
+import { authGoogleToken } from '@/utils/plugin/google';
+import requestIp from 'request-ip';

 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
  try {
-    const { username, type } = req.query as { username: string; type: `${UserAuthTypeEnum}` };
+    const { username, type, googleToken } = req.body as {
+      username: string;
+      type: `${UserAuthTypeEnum}`;
+      googleToken: string;
+    };

    if (!username || !type) {
      throw new Error('缺少参数');
    }

+    // google auth
+    process.env.SERVICE_GOOGLE_VER_TOKEN &&
+      (await authGoogleToken({
+        secret: process.env.SERVICE_GOOGLE_VER_TOKEN,
+        response: googleToken,
+        remoteip: requestIp.getClientIp(req) || undefined
+      }));
+
    await connectToDatabase();

    const code = nanoid();
--- a/src/pages/chat/index.tsx
+++ b/src/pages/chat/index.tsx
@@ -291,7 +291,7 @@ const Chat = ({ modelId, chatId }: { modelId: string; chatId: string }) => {
    }, 100);

    try {
-      await gptChatPrompt(newChatList.slice(-2));
+      await gptChatPrompt(newChatList.slice(newChatList.length - 2));
    } catch (err: any) {
      toast({
        title: typeof err === 'string' ? err : err?.message || '聊天出错了~',
--- a/src/pages/index.tsx
+++ b/src/pages/index.tsx
@@ -2,8 +2,6 @@ import React, { useEffect } from 'react';
 import { Card, Box, Link, Flex, Image, Button } from '@chakra-ui/react';
 import Markdown from '@/components/Markdown';
 import { useMarkdown } from '@/hooks/useMarkdown';
-import { getFilling } from '@/api/system';
-import { useQuery } from '@tanstack/react-query';
 import { useRouter } from 'next/router';
 import { useGlobalStore } from '@/store/global';

@@ -13,7 +11,10 @@ const Home = () => {
  const router = useRouter();
  const { inviterId } = router.query as { inviterId: string };
  const { data } = useMarkdown({ url: '/intro.md' });
-  const { isPc } = useGlobalStore();
+  const {
+    isPc,
+    initData: { beianText }
+  } = useGlobalStore();

  useEffect(() => {
    if (inviterId) {
@@ -21,8 +22,6 @@ const Home = () => {
    }
  }, [inviterId]);

-  const { data: { beianText = '' } = {} } = useQuery(['init'], getFilling);
-
  /* 加载动画 */
  useEffect(() => {
    setTimeout(() => {
--- a/src/pages/kb/components/DataCard.tsx
+++ b/src/pages/kb/components/DataCard.tsx
@@ -22,7 +22,6 @@ import {
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
 import type { BoxProps } from '@chakra-ui/react';
 import type { KbDataItemType } from '@/types/plugin';
-import { ModelDataStatusMap } from '@/constants/model';
 import { usePagination } from '@/hooks/usePagination';
 import {
  getKbDataList,
@@ -91,9 +90,9 @@ const DataCard = ({ kbId }: { kbId: string }) => {
    onClose: onCloseSelectCsvModal
  } = useDisclosure();

-  const { data: { splitDataQueue = 0, embeddingQueue = 0 } = {}, refetch } = useQuery(
-    ['getModelSplitDataList'],
-    () => getTrainingData(kbId),
+  const { data: { qaListLen = 0, vectorListLen = 0 } = {}, refetch } = useQuery(
+    ['getModelSplitDataList', kbId],
+    () => getTrainingData({ kbId, init: false }),
    {
      onError(err) {
        console.log(err);
@@ -113,7 +112,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
  // interval get data
  useQuery(['refetchData'], () => refetchData(pageNum), {
    refetchInterval: 5000,
-    enabled: splitDataQueue > 0 || embeddingQueue > 0
+    enabled: qaListLen > 0 || vectorListLen > 0
  });

  // get al data and export csv
@@ -161,7 +160,10 @@ const DataCard = ({ kbId }: { kbId: string }) => {
          variant={'outline'}
          mr={[2, 4]}
          size={'sm'}
-          onClick={() => refetchData(pageNum)}
+          onClick={() => {
+            refetchData(pageNum);
+            getTrainingData({ kbId, init: true });
+          }}
        />
        <Button
          variant={'outline'}
@@ -194,10 +196,10 @@ const DataCard = ({ kbId }: { kbId: string }) => {
        </Menu>
      </Flex>
      <Flex mt={4}>
-        {(splitDataQueue > 0 || embeddingQueue > 0) && (
+        {(qaListLen > 0 || vectorListLen > 0) && (
          <Box fontSize={'xs'}>
-            {splitDataQueue > 0 ? `${splitDataQueue}条数据正在拆分，` : ''}
-            {embeddingQueue > 0 ? `${embeddingQueue}条数据正在生成索引，` : ''}
+            {qaListLen > 0 ? `${qaListLen}条数据正在拆分，` : ''}
+            {vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引，` : ''}
            请耐心等待...
          </Box>
        )}
@@ -237,7 +239,6 @@ const DataCard = ({ kbId }: { kbId: string }) => {
                </Tooltip>
              </Th>
              <Th>补充知识</Th>
-              <Th>状态</Th>
              <Th>操作</Th>
            </Tr>
          </Thead>
@@ -250,7 +251,6 @@ const DataCard = ({ kbId }: { kbId: string }) => {
                <Td>
                  <Box {...tdStyles.current}>{item.a || '-'}</Box>
                </Td>
-                <Td>{ModelDataStatusMap[item.status]}</Td>
                <Td>
                  <IconButton
                    mr={5}
--- a/src/pages/kb/components/Detail.tsx
+++ b/src/pages/kb/components/Detail.tsx
@@ -56,13 +56,13 @@ const Detail = ({ kbId }: { kbId: string }) => {
      }
    },
    onError(err: any) {
+      loadKbList(true);
+      setLastKbId('');
+      router.replace(`/kb`);
      toast({
        title: getErrText(err, '获取知识库异常'),
        status: 'error'
      });
-      loadKbList(true);
-      setLastKbId('');
-      router.replace(`/kb?kbId=${myKbList[0]?._id || ''}`);
    }
  });

--- a/src/pages/kb/components/InputDataModal.tsx
+++ b/src/pages/kb/components/InputDataModal.tsx
@@ -13,6 +13,8 @@ import {
 import { useForm } from 'react-hook-form';
 import { postKbDataFromList, putKbDataById } from '@/api/plugins/kb';
 import { useToast } from '@/hooks/useToast';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { getErrText } from '@/utils/tools';

 export type FormData = { dataId?: string; a: string; q: string };

@@ -52,31 +54,39 @@ const InputDataModal = ({
      setLoading(true);

      try {
-        const res = await postKbDataFromList({
+        const { insertLen } = await postKbDataFromList({
          kbId,
          data: [
            {
              a: e.a,
              q: e.q
            }
-          ]
+          ],
+          mode: TrainingModeEnum.index
        });

-        toast({
-          title: res === 0 ? '可能已存在完全一致的数据' : '导入数据成功,需要一段时间训练',
-          status: 'success'
-        });
-        reset({
-          a: '',
-          q: ''
-        });
+        if (insertLen === 0) {
+          toast({
+            title: '已存在完全一致的数据',
+            status: 'warning'
+          });
+        } else {
+          toast({
+            title: '导入数据成功,需要一段时间训练',
+            status: 'success'
+          });
+          reset({
+            a: '',
+            q: ''
+          });
+        }
+
        onSuccess();
      } catch (err: any) {
        toast({
-          title: err?.message || '出现了点意外~',
+          title: getErrText(err, '出现了点意外~'),
          status: 'error'
        });
-        console.log(err);
      }
      setLoading(false);
    },
@@ -124,7 +134,8 @@ const InputDataModal = ({
        <ModalCloseButton />

        <Box
-          display={['block', 'flex']}
+          display={'flex'}
+          flexDirection={['column', 'row']}
          flex={'1 0 0'}
          h={['100%', 0]}
          overflow={'overlay'}
--- a/src/pages/kb/components/SelectCsvModal.tsx
+++ b/src/pages/kb/components/SelectCsvModal.tsx
@@ -19,6 +19,8 @@ import { postKbDataFromList } from '@/api/plugins/kb';
 import Markdown from '@/components/Markdown';
 import { useMarkdown } from '@/hooks/useMarkdown';
 import { fileDownload } from '@/utils/file';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { getErrText } from '@/utils/tools';

 const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;

@@ -35,6 +37,7 @@ const SelectJsonModal = ({
  const { toast } = useToast();
  const { File, onOpen } = useSelectFile({ fileType: '.csv', multiple: false });
  const [fileData, setFileData] = useState<{ q: string; a: string }[]>([]);
+  const [successData, setSuccessData] = useState(0);
  const { openConfirm, ConfirmChild } = useConfirm({
    content: '确认导入该数据集?'
  });
@@ -55,9 +58,8 @@ const SelectJsonModal = ({
          }))
        );
      } catch (error: any) {
-        console.log(error);
        toast({
-          title: error?.message || 'csv 文件格式有误',
+          title: getErrText(error, 'csv 文件格式有误'),
          status: 'error'
        });
      }
@@ -66,26 +68,35 @@ const SelectJsonModal = ({
    [setSelecting, toast]
  );

-  const { mutate, isLoading } = useMutation({
+  const { mutate, isLoading: uploading } = useMutation({
    mutationFn: async () => {
      if (!fileData || fileData.length === 0) return;

-      const res = await postKbDataFromList({
-        kbId,
-        data: fileData
-      });
+      let success = 0;
+
+      // subsection import
+      const step = 50;
+      for (let i = 0; i < fileData.length; i += step) {
+        const { insertLen } = await postKbDataFromList({
+          kbId,
+          data: fileData.slice(i, i + step),
+          mode: TrainingModeEnum.index
+        });
+        success += insertLen || 0;
+        setSuccessData((state) => state + step);
+      }

      toast({
-        title: `导入数据成功，最终导入: ${res || 0} 条数据。需要一段时间训练`,
+        title: `导入数据成功，最终导入: ${success} 条数据。需要一段时间训练`,
        status: 'success',
        duration: 4000
      });
      onClose();
      onSuccess();
    },
-    onError() {
+    onError(err) {
      toast({
-        title: '导入文件失败',
+        title: getErrText(err, '导入文件失败'),
        status: 'error'
      });
    }
@@ -119,15 +130,15 @@ const SelectJsonModal = ({
              点击下载csv模板
            </Box>
            <Flex alignItems={'center'}>
-              <Button isLoading={selecting} onClick={onOpen}>
+              <Button isLoading={selecting} isDisabled={uploading} onClick={onOpen}>
                选择 csv 问答对
              </Button>

-              <Box ml={4}>一共 {fileData.length} 组数据</Box>
+              <Box ml={4}>一共 {fileData.length} 组数据（下面最多展示100组）</Box>
            </Flex>
          </Box>
          <Box flex={'3 0 0'} h={'100%'} overflow={'auto'} p={2} backgroundColor={'blackAlpha.50'}>
-            {fileData.map((item, index) => (
+            {fileData.slice(0, 100).map((item, index) => (
              <Box key={index}>
                <Box>
                  Q{index + 1}. {item.q}
@@ -142,15 +153,15 @@ const SelectJsonModal = ({

        <Flex px={6} pt={2} pb={4}>
          <Box flex={1}></Box>
-          <Button variant={'outline'} mr={3} onClick={onClose}>
+          <Button variant={'outline'} isLoading={uploading} mr={3} onClick={onClose}>
            取消
          </Button>
-          <Button
-            isLoading={isLoading}
-            isDisabled={fileData.length === 0}
-            onClick={openConfirm(mutate)}
-          >
-            确认导入
+          <Button isDisabled={fileData.length === 0 || uploading} onClick={openConfirm(mutate)}>
+            {uploading ? (
+              <Box>{Math.round((successData / fileData.length) * 100)}%</Box>
+            ) : (
+              '确认导入'
+            )}
          </Button>
        </Flex>
      </ModalContent>
--- a/src/pages/kb/components/SelectFileModal.tsx
+++ b/src/pages/kb/components/SelectFileModal.tsx
@@ -17,21 +17,22 @@ import { useSelectFile } from '@/hooks/useSelectFile';
 import { useConfirm } from '@/hooks/useConfirm';
 import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
 import { useMutation } from '@tanstack/react-query';
-import { postSplitData } from '@/api/plugins/kb';
+import { postKbDataFromList } from '@/api/plugins/kb';
 import Radio from '@/components/Radio';
 import { splitText_token } from '@/utils/file';
-import { SplitTextTypEnum } from '@/constants/plugin';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { getErrText } from '@/utils/tools';

 const fileExtension = '.txt,.doc,.docx,.pdf,.md';

 const modeMap = {
-  qa: {
+  [TrainingModeEnum.qa]: {
    maxLen: 2800,
    slideLen: 800,
    price: 4,
    isPrompt: true
  },
-  subsection: {
+  [TrainingModeEnum.index]: {
    maxLen: 800,
    slideLen: 300,
    price: 0.4,
@@ -52,11 +53,16 @@ const SelectFileModal = ({
  const { toast } = useToast();
  const [prompt, setPrompt] = useState('');
  const { File, onOpen } = useSelectFile({ fileType: fileExtension, multiple: true });
-  const [mode, setMode] = useState<`${SplitTextTypEnum}`>(SplitTextTypEnum.subsection);
+  const [mode, setMode] = useState<`${TrainingModeEnum}`>(TrainingModeEnum.index);
  const [fileTextArr, setFileTextArr] = useState<string[]>(['']);
-  const [splitRes, setSplitRes] = useState<{ tokens: number; chunks: string[] }>({
+  const [splitRes, setSplitRes] = useState<{
+    tokens: number;
+    chunks: string[];
+    successChunks: number;
+  }>({
    tokens: 0,
-    chunks: []
+    chunks: [],
+    successChunks: 0
  });
  const { openConfirm, ConfirmChild } = useConfirm({
    content: `确认导入该文件，需要一定时间进行拆解，该任务无法终止！如果余额不足，未完成的任务会被直接清除。一共 ${
@@ -103,26 +109,38 @@ const SelectFileModal = ({
    [toast]
  );

-  const { mutate, isLoading } = useMutation({
+  const { mutate, isLoading: uploading } = useMutation({
    mutationFn: async () => {
      if (splitRes.chunks.length === 0) return;

-      await postSplitData({
-        kbId,
-        chunks: splitRes.chunks,
-        prompt: `下面是"${prompt || '一段长文本'}"`,
-        mode
-      });
+      // subsection import
+      let success = 0;
+      const step = 50;
+      for (let i = 0; i < splitRes.chunks.length; i += step) {
+        const { insertLen } = await postKbDataFromList({
+          kbId,
+          data: splitRes.chunks.slice(i, i + step).map((text) => ({ q: text, a: '' })),
+          prompt: `下面是"${prompt || '一段长文本'}"`,
+          mode
+        });
+
+        success += insertLen;
+        setSplitRes((state) => ({
+          ...state,
+          successChunks: state.successChunks + step
+        }));
+      }
+
      toast({
-        title: '导入数据成功,需要一段拆解和训练',
+        title: `去重后共导入 ${success} 条数据,需要一段拆解和训练.`,
        status: 'success'
      });
      onClose();
      onSuccess();
    },
-    onError() {
+    onError(err) {
      toast({
-        title: '导入文件失败',
+        title: getErrText(err, '导入文件失败'),
        status: 'error'
      });
    }
@@ -130,27 +148,36 @@ const SelectFileModal = ({

  const onclickImport = useCallback(async () => {
    setBtnLoading(true);
-    let promise = Promise.resolve();
+    try {
+      let promise = Promise.resolve();

-    const splitRes = fileTextArr
-      .filter((item) => item)
-      .map((item) =>
-        splitText_token({
-          text: item,
-          ...modeMap[mode]
-        })
+      const splitRes = await Promise.all(
+        fileTextArr
+          .filter((item) => item)
+          .map((item) =>
+            splitText_token({
+              text: item,
+              ...modeMap[mode]
+            })
+          )
      );

-    setSplitRes({
-      tokens: splitRes.reduce((sum, item) => sum + item.tokens, 0),
-      chunks: splitRes.map((item) => item.chunks).flat()
-    });
+      setSplitRes({
+        tokens: splitRes.reduce((sum, item) => sum + item.tokens, 0),
+        chunks: splitRes.map((item) => item.chunks).flat(),
+        successChunks: 0
+      });

+      await promise;
+      openConfirm(mutate)();
+    } catch (error) {
+      toast({
+        status: 'warning',
+        title: getErrText(error, '拆分文本异常')
+      });
+    }
    setBtnLoading(false);
-
-    await promise;
-    openConfirm(mutate)();
-  }, [fileTextArr, mode, mutate, openConfirm]);
+  }, [fileTextArr, mode, mutate, openConfirm, toast]);

  return (
    <Modal isOpen={true} onClose={onClose} isCentered>
@@ -185,11 +212,11 @@ const SelectFileModal = ({
            <Radio
              ml={3}
              list={[
-                { label: '直接分段', value: 'subsection' },
+                { label: '直接分段', value: 'index' },
                { label: 'QA拆分', value: 'qa' }
              ]}
              value={mode}
-              onChange={(e) => setMode(e as 'subsection' | 'qa')}
+              onChange={(e) => setMode(e as 'index' | 'qa')}
            />
          </Flex>
          {/* 内容介绍 */}
@@ -225,6 +252,11 @@ const SelectFileModal = ({
                      ...fileTextArr.slice(i + 1)
                    ]);
                  }}
+                  onBlur={(e) => {
+                    if (fileTextArr.length > 1 && e.target.value === '') {
+                      setFileTextArr((state) => [...state.slice(0, i), ...state.slice(i + 1)]);
+                    }
+                  }}
                />
              </Box>
            ))}
@@ -232,19 +264,22 @@ const SelectFileModal = ({
        </ModalBody>

        <Flex px={6} pt={2} pb={4}>
-          <Button isLoading={btnLoading} onClick={onOpen}>
+          <Button isLoading={btnLoading} isDisabled={uploading} onClick={onOpen}>
            选择文件
          </Button>
          <Box flex={1}></Box>
-          <Button variant={'outline'} colorScheme={'gray'} mr={3} onClick={onClose}>
+          <Button variant={'outline'} isLoading={uploading} mr={3} onClick={onClose}>
            取消
          </Button>
          <Button
-            isLoading={isLoading || btnLoading}
-            isDisabled={isLoading || btnLoading || fileTextArr[0] === ''}
+            isDisabled={uploading || btnLoading || fileTextArr[0] === ''}
            onClick={onclickImport}
          >
-            确认导入
+            {uploading ? (
+              <Box>{Math.round((splitRes.successChunks / splitRes.chunks.length) * 100)}%</Box>
+            ) : (
+              '确认导入'
+            )}
          </Button>
        </Flex>
      </ModalContent>
--- a/src/service/api/request.ts
+++ b/src/service/api/request.ts
@@ -53,10 +53,11 @@ function responseError(err: any) {
 }

 /* 创建请求实例 */
-const instance = axios.create({
+export const instance = axios.create({
  timeout: 60000, // 超时时间
+  baseURL: `http://localhost:${process.env.PORT || 3000}/api`,
  headers: {
-    'content-type': 'application/json'
+    rootkey: process.env.ROOT_KEY
  }
 });

@@ -75,7 +76,6 @@ function request(url: string, data: any, config: ConfigType, method: Method): an

  return instance
    .request({
-      baseURL: `http://localhost:${process.env.PORT || 3000}/api`,
      url,
      method,
      data: method === 'GET' ? null : data,
@@ -93,18 +93,30 @@ function request(url: string, data: any, config: ConfigType, method: Method): an
 * @param {Object} config
 * @returns
 */
-export function GET<T>(url: string, params = {}, config: ConfigType = {}): Promise<T> {
+export function GET<T = { data: any }>(
+  url: string,
+  params = {},
+  config: ConfigType = {}
+): Promise<T> {
  return request(url, params, config, 'GET');
 }

-export function POST<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function POST<T = { data: any }>(
+  url: string,
+  data = {},
+  config: ConfigType = {}
+): Promise<T> {
  return request(url, data, config, 'POST');
 }

-export function PUT<T>(url: string, data = {}, config: ConfigType = {}): Promise<T> {
+export function PUT<T = { data: any }>(
+  url: string,
+  data = {},
+  config: ConfigType = {}
+): Promise<T> {
  return request(url, data, config, 'PUT');
 }

-export function DELETE<T>(url: string, config: ConfigType = {}): Promise<T> {
+export function DELETE<T = { data: any }>(url: string, config: ConfigType = {}): Promise<T> {
  return request(url, {}, config, 'DELETE');
 }
--- a/src/service/errorCode.ts
+++ b/src/service/errorCode.ts
@@ -24,10 +24,10 @@ export const openaiError: Record<string, string> = {
  'Bad Request': 'Bad Request~ 可能内容太多了',
  'Bad Gateway': '网关异常，请重试'
 };
-export const openaiError2: Record<string, string> = {
+export const openaiAccountError: Record<string, string> = {
  insufficient_quota: 'API 余额不足',
-  billing_not_active: 'openai 账号异常',
-  invalid_request_error: '无效的 openai 请求'
+  invalid_api_key: 'openai 账号异常',
+  account_deactivated: '账号已停用'
 };
 export const proxyError: Record<string, boolean> = {
  ECONNABORTED: true,
--- a/src/service/events/generateQA.ts
+++ b/src/service/events/generateQA.ts
@@ -1,75 +1,91 @@
-import { SplitData } from '@/service/mongo';
+import { TrainingData } from '@/service/mongo';
 import { getApiKey } from '../utils/auth';
 import { OpenAiChatEnum } from '@/constants/model';
 import { pushSplitDataBill } from '@/service/events/pushBill';
-import { generateVector } from './generateVector';
-import { openaiError2 } from '../errorCode';
-import { insertKbItem } from '@/service/pg';
-import { SplitDataSchema } from '@/types/mongoSchema';
+import { openaiAccountError } from '../errorCode';
 import { modelServiceToolMap } from '../utils/chat';
 import { ChatRoleEnum } from '@/constants/chat';
-import { getErrText } from '@/utils/tools';
 import { BillTypeEnum } from '@/constants/user';
+import { pushDataToKb } from '@/pages/api/openapi/kb/pushData';
+import { TrainingModeEnum } from '@/constants/plugin';
+import { ERROR_ENUM } from '../errorCode';

-export async function generateQA(next = false): Promise<any> {
-  if (process.env.queueTask !== '1') {
-    try {
-      fetch(process.env.parentUrl || '');
-    } catch (error) {
-      console.log('parentUrl fetch error', error);
-    }
-    return;
-  }
-  if (global.generatingQA === true && !next) return;
+const reduceQueue = () => {
+  global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
+};

-  global.generatingQA = true;
+export async function generateQA(): Promise<any> {
+  const maxProcess = Number(process.env.QA_MAX_PROCESS || 10);

-  let dataId = null;
+  if (global.qaQueueLen >= maxProcess) return;
+  global.qaQueueLen++;
+
+  let trainingId = '';
+  let userId = '';

  try {
-    // 找出一个需要生成的 dataItem
-    const data = await SplitData.aggregate([
-      { $match: { textList: { $exists: true, $ne: [] } } },
-      { $sample: { size: 1 } }
+    const match = {
+      mode: TrainingModeEnum.qa,
+      lockTime: { $lte: new Date(Date.now() - 4 * 60 * 1000) }
+    };
+    // random get task
+    const agree = await TrainingData.aggregate([
+      {
+        $match: match
+      },
+      { $sample: { size: 1 } },
+      {
+        $project: {
+          _id: 1
+        }
+      }
    ]);

-    const dataItem: SplitDataSchema = data[0];
-
-    if (!dataItem) {
-      console.log('没有需要生成 QA 的数据');
-      global.generatingQA = false;
+    // no task
+    if (agree.length === 0) {
+      reduceQueue();
+      global.qaQueueLen <= 0 && console.log(`没有需要【QA】的数据, ${global.qaQueueLen}`);
      return;
    }

-    dataId = dataItem._id;
+    const data = await TrainingData.findOneAndUpdate(
+      {
+        _id: agree[0]._id,
+        ...match
+      },
+      {
+        lockTime: new Date()
+      }
+    ).select({
+      _id: 1,
+      userId: 1,
+      kbId: 1,
+      prompt: 1,
+      q: 1
+    });

-    // 获取 5 个源文本
-    const textList: string[] = dataItem.textList.slice(-5);
-
-    // 获取 openapi Key
-    let userOpenAiKey = '',
-      systemAuthKey = '';
-    try {
-      const key = await getApiKey({ model: OpenAiChatEnum.GPT35, userId: dataItem.userId });
-      userOpenAiKey = key.userOpenAiKey;
-      systemAuthKey = key.systemAuthKey;
-    } catch (err: any) {
-      // 余额不够了, 清空该记录
-      await SplitData.findByIdAndUpdate(dataItem._id, {
-        textList: [],
-        errorText: getErrText(err, '获取 OpenAi Key 失败')
-      });
-      generateQA(true);
-      return;
+    // task preemption
+    if (!data) {
+      reduceQueue();
+      return generateQA();
    }

-    console.log(`正在生成一组QA, 包含 ${textList.length} 组文本。ID: ${dataItem._id}`);
+    trainingId = data._id;
+    userId = String(data.userId);
+    const kbId = String(data.kbId);
+
+    // 余额校验并获取 openapi Key
+    const { userOpenAiKey, systemAuthKey } = await getApiKey({
+      model: OpenAiChatEnum.GPT35,
+      userId,
+      type: 'training'
+    });

    const startTime = Date.now();

    // 请求 chatgpt 获取回答
-    const response = await Promise.allSettled(
-      textList.map((text) =>
+    const response = await Promise.all(
+      [data.q].map((text) =>
        modelServiceToolMap[OpenAiChatEnum.GPT35]
          .chatCompletion({
            apiKey: userOpenAiKey || systemAuthKey,
@@ -78,7 +94,7 @@ export async function generateQA(next = false): Promise<any> {
              {
                obj: ChatRoleEnum.System,
                value: `你是出题人
-${dataItem.prompt || '下面是"一段长文本"'}
+${data.prompt || '下面是"一段长文本"'}
 从中选出5至20个题目和答案.答案详细.按格式返回: Q1:
 A1:
 Q2:
@@ -98,7 +114,7 @@ A2:
            // 计费
            pushSplitDataBill({
              isPay: !userOpenAiKey && result.length > 0,
-              userId: dataItem.userId,
+              userId: data.userId,
              type: BillTypeEnum.QA,
              textLen: responseMessages.map((item) => item.value).join('').length,
              totalTokens
@@ -116,57 +132,58 @@ A2:
      )
    );

-    // 获取成功的回答
-    const successResponse: {
-      rawContent: string;
-      result: {
-        q: string;
-        a: string;
-      }[];
-    }[] = response.filter((item) => item.status === 'fulfilled').map((item: any) => item.value);
+    const responseList = response.map((item) => item.result).flat();

-    const resultList = successResponse.map((item) => item.result).flat();
+    // 创建 向量生成 队列
+    await pushDataToKb({
+      kbId,
+      data: responseList,
+      userId,
+      mode: TrainingModeEnum.index
+    });
+
+    // delete data from training
+    await TrainingData.findByIdAndDelete(data._id);

-    await Promise.allSettled([
-      // 删掉后5个数据
-      SplitData.findByIdAndUpdate(dataItem._id, {
-        textList: dataItem.textList.slice(0, -5)
-      }),
-      // 生成的内容插入 pg
-      insertKbItem({
-        userId: dataItem.userId,
-        kbId: dataItem.kbId,
-        data: resultList
-      })
-    ]);
    console.log('生成QA成功，time:', `${(Date.now() - startTime) / 1000}s`);

-    generateQA(true);
-    generateVector();
-  } catch (error: any) {
+    reduceQueue();
+    generateQA();
+  } catch (err: any) {
+    reduceQueue();
    // log
-    if (error?.response) {
+    if (err?.response) {
      console.log('openai error: 生成QA错误');
-      console.log(error.response?.status, error.response?.statusText, error.response?.data);
+      console.log(err.response?.status, err.response?.statusText, err.response?.data);
    } else {
-      console.log('生成QA错误:', error);
+      console.log('生成QA错误:', err);
    }

-    // 没有余额或者凭证错误时，拒绝任务
-    if (dataId && openaiError2[error?.response?.data?.error?.type]) {
-      console.log(openaiError2[error?.response?.data?.error?.type], '删除QA任务');
+    // message error or openai account error
+    if (
+      err?.message === 'invalid message format' ||
+      err.response?.statusText === 'Unauthorized' ||
+      openaiAccountError[err?.response?.data?.error?.code || err?.response?.data?.error?.type]
+    ) {
+      await TrainingData.findByIdAndRemove(trainingId);
+    }

-      await SplitData.findByIdAndUpdate(dataId, {
-        textList: [],
-        errorText: 'api 余额不足'
+    // 账号余额不足，删除任务
+    if (err === ERROR_ENUM.insufficientQuota) {
+      console.log('余额不足，删除向量生成任务');
+      await TrainingData.deleteMany({
+        userId
      });
-
-      generateQA(true);
-      return;
+      return generateQA();
    }

+    // unlock
+    await TrainingData.findByIdAndUpdate(trainingId, {
+      lockTime: new Date('2000/1/1')
+    });
+
    setTimeout(() => {
-      generateQA(true);
+      generateQA();
    }, 1000);
  }
 }
--- a/src/service/events/generateVector.ts
+++ b/src/service/events/generateVector.ts
@@ -1,107 +1,141 @@
-import { getApiKey } from '../utils/auth';
-import { openaiError2 } from '../errorCode';
-import { PgClient } from '@/service/pg';
-import { getErrText } from '@/utils/tools';
+import { openaiAccountError } from '../errorCode';
+import { insertKbItem } from '@/service/pg';
 import { openaiEmbedding } from '@/pages/api/openapi/plugin/openaiEmbedding';
+import { TrainingData } from '../models/trainingData';
+import { ERROR_ENUM } from '../errorCode';
+import { TrainingModeEnum } from '@/constants/plugin';

-export async function generateVector(next = false): Promise<any> {
-  if (process.env.queueTask !== '1') {
-    try {
-      fetch(process.env.parentUrl || '');
-    } catch (error) {
-      console.log('parentUrl fetch error', error);
-    }
-    return;
-  }
+const reduceQueue = () => {
+  global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0;
+};

-  if (global.generatingVector && !next) return;
+/* 索引生成队列。每导入一次，就是一个单独的线程 */
+export async function generateVector(): Promise<any> {
+  const maxProcess = Number(process.env.VECTOR_MAX_PROCESS || 10);

-  global.generatingVector = true;
-  let dataId = null;
+  if (global.vectorQueueLen >= maxProcess) return;
+  global.vectorQueueLen++;
+
+  let trainingId = '';
+  let userId = '';

  try {
-    // 从找出一个 status = waiting 的数据
-    const searchRes = await PgClient.select('modelData', {
-      fields: ['id', 'q', 'user_id'],
-      where: [['status', 'waiting']],
-      limit: 1
-    });
+    const match = {
+      mode: TrainingModeEnum.index,
+      lockTime: { $lte: new Date(Date.now() - 2 * 60 * 1000) }
+    };
+    // random get task
+    const agree = await TrainingData.aggregate([
+      {
+        $match: match
+      },
+      { $sample: { size: 1 } },
+      {
+        $project: {
+          _id: 1
+        }
+      }
+    ]);

-    if (searchRes.rowCount === 0) {
-      console.log('没有需要生成 【向量】 的数据');
-      global.generatingVector = false;
+    // no task
+    if (agree.length === 0) {
+      reduceQueue();
+      global.vectorQueueLen <= 0 && console.log(`没有需要【索引】的数据, ${global.vectorQueueLen}`);
      return;
    }

-    const dataItem: { id: string; q: string; userId: string } = {
-      id: searchRes.rows[0].id,
-      q: searchRes.rows[0].q,
-      userId: searchRes.rows[0].user_id
-    };
+    const data = await TrainingData.findOneAndUpdate(
+      {
+        _id: agree[0]._id,
+        ...match
+      },
+      {
+        lockTime: new Date()
+      }
+    ).select({
+      _id: 1,
+      userId: 1,
+      kbId: 1,
+      q: 1,
+      a: 1
+    });

-    dataId = dataItem.id;
-
-    // 获取 openapi Key
-    try {
-      await getApiKey({ model: 'gpt-3.5-turbo', userId: dataItem.userId });
-    } catch (err: any) {
-      await PgClient.delete('modelData', {
-        where: [['id', dataId]]
-      });
-      getErrText(err, '获取 OpenAi Key 失败');
-      return generateVector(true);
+    // task preemption
+    if (!data) {
+      reduceQueue();
+      return generateVector();
    }

+    trainingId = data._id;
+    userId = String(data.userId);
+    const kbId = String(data.kbId);
+
+    const dataItems = [
+      {
+        q: data.q,
+        a: data.a
+      }
+    ];
+
    // 生成词向量
    const vectors = await openaiEmbedding({
-      input: [dataItem.q],
-      userId: dataItem.userId
+      input: dataItems.map((item) => item.q),
+      userId,
+      type: 'training'
    });

-    // 更新 pg 向量和状态数据
-    await PgClient.update('modelData', {
-      values: [
-        { key: 'vector', value: `[${vectors[0]}]` },
-        { key: 'status', value: `ready` }
-      ],
-      where: [['id', dataId]]
+    // 生成结果插入到 pg
+    await insertKbItem({
+      userId,
+      kbId,
+      data: vectors.map((vector, i) => ({
+        q: dataItems[i].q,
+        a: dataItems[i].a,
+        vector
+      }))
    });

-    console.log(`生成向量成功: ${dataItem.id}`);
+    // delete data from training
+    await TrainingData.findByIdAndDelete(data._id);
+    console.log(`生成向量成功: ${data._id}`);

-    generateVector(true);
-  } catch (error: any) {
+    reduceQueue();
+    generateVector();
+  } catch (err: any) {
+    reduceQueue();
    // log
-    if (error?.response) {
+    if (err?.response) {
      console.log('openai error: 生成向量错误');
-      console.log(error.response?.status, error.response?.statusText, error.response?.data);
+      console.log(err.response?.status, err.response?.statusText, err.response?.data);
    } else {
-      console.log('生成向量错误:', error);
+      console.log('生成向量错误:', err);
+    }
+    // message error or openai account error
+    if (
+      err?.message === 'invalid message format' ||
+      err.response?.statusText === 'Unauthorized' ||
+      openaiAccountError[err?.response?.data?.error?.code || err?.response?.data?.error?.type]
+    ) {
+      console.log('删除一个任务');
+      await TrainingData.findByIdAndRemove(trainingId);
    }

-    // 没有余额或者凭证错误时，拒绝任务
-    if (dataId && openaiError2[error?.response?.data?.error?.type]) {
-      console.log('删除向量生成任务记录');
-      try {
-        await PgClient.delete('modelData', {
-          where: [['id', dataId]]
-        });
-      } catch (error) {
-        error;
-      }
-      generateVector(true);
-      return;
-    }
-    if (error?.response?.statusText === 'Too Many Requests') {
-      console.log('生成向量次数限制，1分钟后尝试');
-      // 限制次数，1分钟后再试
-      setTimeout(() => {
-        generateVector(true);
-      }, 60000);
-      return;
+    // 账号余额不足，删除任务
+    if (err === ERROR_ENUM.insufficientQuota) {
+      console.log('余额不足，删除向量生成任务');
+      await TrainingData.deleteMany({
+        userId
+      });
+      return generateVector();
    }
+
+    // unlock
+    await TrainingData.findByIdAndUpdate(trainingId, {
+      lockTime: new Date('2000/1/1')
+    });
+
    setTimeout(() => {
-      generateVector(true);
+      generateVector();
    }, 1000);
  }
 }
--- a/src/service/events/pushBill.ts
+++ b/src/service/events/pushBill.ts
@@ -134,9 +134,9 @@ export const pushGenerateVectorBill = async ({
  text: string;
  tokenLen: number;
 }) => {
-  console.log(
-    `vector generate success. text len: ${text.length}. token len: ${tokenLen}. pay:${isPay}`
-  );
+  // console.log(
+  //   `vector generate success. text len: ${text.length}. token len: ${tokenLen}. pay:${isPay}`
+  // );
  if (!isPay) return;

  let billId;
--- a/src/service/models/chat.ts
+++ b/src/service/models/chat.ts
@@ -50,22 +50,10 @@ const ChatSchema = new Schema({
        quote: {
          type: [
            {
-              id: {
-                type: String,
-                required: true
-              },
-              q: {
-                type: String,
-                default: ''
-              },
-              a: {
-                type: String,
-                default: ''
-              },
-              isEdit: {
-                type: String,
-                default: false
-              }
+              id: String,
+              q: String,
+              a: String,
+              isEdit: Boolean
            }
          ],
          default: []
--- a/src/service/models/model.ts
+++ b/src/service/models/model.ts
@@ -2,7 +2,7 @@ import { Schema, model, models, Model as MongoModel } from 'mongoose';
 import { ModelSchema as ModelType } from '@/types/mongoSchema';
 import {
  ModelVectorSearchModeMap,
-  ModelVectorSearchModeEnum,
+  appVectorSearchModeEnum,
  ChatModelMap,
  OpenAiChatEnum
 } from '@/constants/model';
@@ -40,7 +40,7 @@ const ModelSchema = new Schema({
      // knowledge base search mode
      type: String,
      enum: Object.keys(ModelVectorSearchModeMap),
-      default: ModelVectorSearchModeEnum.hightSimilarity
+      default: appVectorSearchModeEnum.hightSimilarity
    },
    systemPrompt: {
      // 系统提示词
--- a/src/service/models/splitData.ts
+++ b/src/service/models/splitData.ts
@@ -1,32 +0,0 @@
-/* 模型的知识库 */
-import { Schema, model, models, Model as MongoModel } from 'mongoose';
-import { SplitDataSchema as SplitDataType } from '@/types/mongoSchema';
-
-const SplitDataSchema = new Schema({
-  userId: {
-    type: Schema.Types.ObjectId,
-    ref: 'user',
-    required: true
-  },
-  prompt: {
-    // 拆分时的提示词
-    type: String,
-    required: true
-  },
-  kbId: {
-    type: Schema.Types.ObjectId,
-    ref: 'kb',
-    required: true
-  },
-  textList: {
-    type: [String],
-    default: []
-  },
-  errorText: {
-    type: String,
-    default: ''
-  }
-});
-
-export const SplitData: MongoModel<SplitDataType> =
-  models['splitData'] || model('splitData', SplitDataSchema);
--- a/src/service/models/trainingData.ts
+++ b/src/service/models/trainingData.ts
@@ -0,0 +1,47 @@
+/* 模型的知识库 */
+import { Schema, model, models, Model as MongoModel } from 'mongoose';
+import { TrainingDataSchema as TrainingDateType } from '@/types/mongoSchema';
+import { TrainingTypeMap } from '@/constants/plugin';
+
+// pgList and vectorList, Only one of them will work
+const TrainingDataSchema = new Schema({
+  userId: {
+    type: Schema.Types.ObjectId,
+    ref: 'user',
+    required: true
+  },
+  kbId: {
+    type: Schema.Types.ObjectId,
+    ref: 'kb',
+    required: true
+  },
+  lockTime: {
+    type: Date,
+    default: () => new Date('2000/1/1')
+  },
+  mode: {
+    type: String,
+    enum: Object.keys(TrainingTypeMap),
+    required: true
+  },
+  prompt: {
+    // 拆分时的提示词
+    type: String,
+    default: ''
+  },
+  q: {
+    // 如果是
+    type: String,
+    default: ''
+  },
+  a: {
+    type: String,
+    default: ''
+  },
+  vectorList: {
+    type: Object
+  }
+});
+
+export const TrainingData: MongoModel<TrainingDateType> =
+  models['trainingData'] || model('trainingData', TrainingDataSchema);
--- a/src/service/mongo.ts
+++ b/src/service/mongo.ts
@@ -1,7 +1,7 @@
 import mongoose from 'mongoose';
-import { generateQA } from './events/generateQA';
-import { generateVector } from './events/generateVector';
 import tunnel from 'tunnel';
+import { TrainingData } from './mongo';
+import { startQueue } from './utils/tools';

 /**
 * 连接 MongoDB 数据库
@@ -27,9 +27,6 @@ export async function connectToDatabase(): Promise<void> {
    global.mongodb = null;
  }

-  generateQA();
-  generateVector();
-
  // 创建代理对象
  if (process.env.AXIOS_PROXY_HOST && process.env.AXIOS_PROXY_PORT) {
    global.httpsAgent = tunnel.httpsOverHttp({
@@ -39,6 +36,23 @@ export async function connectToDatabase(): Promise<void> {
      }
    });
  }
+
+  global.qaQueueLen = 0;
+  global.vectorQueueLen = 0;
+
+  startQueue();
+  // 5 分钟后解锁不正常的数据，并触发开始训练
+  setTimeout(async () => {
+    await TrainingData.updateMany(
+      {
+        lockTime: { $lte: Date.now() - 5 * 60 * 1000 }
+      },
+      {
+        lockTime: new Date('2000/1/1')
+      }
+    );
+    startQueue();
+  }, 5 * 60 * 1000);
 }

 export * from './models/authCode';
@@ -47,7 +61,7 @@ export * from './models/model';
 export * from './models/user';
 export * from './models/bill';
 export * from './models/pay';
-export * from './models/splitData';
+export * from './models/trainingData';
 export * from './models/openapi';
 export * from './models/promotionRecord';
 export * from './models/collection';
--- a/src/service/pg.ts
+++ b/src/service/pg.ts
@@ -13,8 +13,8 @@ export const connectPg = async () => {
    password: process.env.PG_PASSWORD,
    database: process.env.PG_DB_NAME,
    max: 20,
-    idleTimeoutMillis: 30000,
-    connectionTimeoutMillis: 2000
+    idleTimeoutMillis: 60000,
+    connectionTimeoutMillis: 20000
  });

  global.pgClient.on('error', (err) => {
@@ -168,6 +168,7 @@ export const insertKbItem = ({
  userId: string;
  kbId: string;
  data: {
+    vector: number[];
    q: string;
    a: string;
  }[];
@@ -176,9 +177,9 @@ export const insertKbItem = ({
    values: data.map((item) => [
      { key: 'user_id', value: userId },
      { key: 'kb_id', value: kbId },
-      { key: 'q', value: item.q },
-      { key: 'a', value: item.a },
-      { key: 'status', value: 'waiting' }
+      { key: 'q', value: item.q.replace(/'/g, '"') },
+      { key: 'a', value: item.a.replace(/'/g, '"') },
+      { key: 'vector', value: `[${item.vector}]` }
    ])
  });
 };
--- a/src/service/response.ts
+++ b/src/service/response.ts
@@ -1,5 +1,11 @@
 import { NextApiResponse } from 'next';
-import { openaiError, openaiError2, proxyError, ERROR_RESPONSE, ERROR_ENUM } from './errorCode';
+import {
+  openaiError,
+  openaiAccountError,
+  proxyError,
+  ERROR_RESPONSE,
+  ERROR_ENUM
+} from './errorCode';
 import { clearCookie } from './utils/tools';

 export interface ResponseType<T = any> {
@@ -40,8 +46,8 @@ export const jsonRes = <T = any>(
      msg = '接口连接异常';
    } else if (error?.response?.data?.error?.message) {
      msg = error?.response?.data?.error?.message;
-    } else if (openaiError2[error?.response?.data?.error?.type]) {
-      msg = openaiError2[error?.response?.data?.error?.type];
+    } else if (openaiAccountError[error?.response?.data?.error?.code]) {
+      msg = openaiAccountError[error?.response?.data?.error?.code];
    } else if (openaiError[error?.response?.statusText]) {
      msg = openaiError[error.response.statusText];
    }
--- a/src/service/utils/auth.ts
+++ b/src/service/utils/auth.ts
@@ -5,12 +5,14 @@ import { Chat, Model, OpenApi, User, ShareChat, KB } from '../mongo';
 import type { ModelSchema } from '@/types/mongoSchema';
 import type { ChatItemSimpleType } from '@/types/chat';
 import mongoose from 'mongoose';
-import { ClaudeEnum, defaultModel } from '@/constants/model';
+import { ClaudeEnum, defaultModel, embeddingModel, EmbeddingModelType } from '@/constants/model';
 import { formatPrice } from '@/utils/user';
 import { ERROR_ENUM } from '../errorCode';
 import { ChatModelType, OpenAiChatEnum } from '@/constants/model';
 import { hashPassword } from '@/service/utils/tools';

+export type ApiKeyType = 'training' | 'chat';
+
 export const parseCookie = (cookie?: string): Promise<string> => {
  return new Promise((resolve, reject) => {
    // 获取 cookie
@@ -118,9 +120,15 @@ export const authUser = async ({
 };

 /* random get openai api key */
-export const getSystemOpenAiKey = () => {
+export const getSystemOpenAiKey = (type: ApiKeyType) => {
+  const keys = (() => {
+    if (type === 'training') {
+      return process.env.OPENAI_TRAINING_KEY?.split(',') || [];
+    }
+    return process.env.OPENAIKEY?.split(',') || [];
+  })();
+
  // 纯字符串类型
-  const keys = process.env.OPENAIKEY?.split(',') || [];
  const i = Math.floor(Math.random() * keys.length);
  return keys[i] || (process.env.OPENAIKEY as string);
 };
@@ -129,11 +137,13 @@ export const getSystemOpenAiKey = () => {
 export const getApiKey = async ({
  model,
  userId,
-  mustPay = false
+  mustPay = false,
+  type = 'chat'
 }: {
  model: ChatModelType;
  userId: string;
  mustPay?: boolean;
+  type?: ApiKeyType;
 }) => {
  const user = await User.findById(userId);
  if (!user) {
@@ -143,7 +153,7 @@ export const getApiKey = async ({
  const keyMap = {
    [OpenAiChatEnum.GPT35]: {
      userOpenAiKey: user.openaiKey || '',
-      systemAuthKey: getSystemOpenAiKey() as string
+      systemAuthKey: getSystemOpenAiKey(type) as string
    },
    [OpenAiChatEnum.GPT4]: {
      userOpenAiKey: user.openaiKey || '',
--- a/src/service/utils/tools.ts
+++ b/src/service/utils/tools.ts
@@ -2,6 +2,8 @@ import type { NextApiResponse, NextApiHandler, NextApiRequest } from 'next';
 import NextCors from 'nextjs-cors';
 import crypto from 'crypto';
 import jwt from 'jsonwebtoken';
+import { generateQA } from '../events/generateQA';
+import { generateVector } from '../events/generateVector';

 /* 密码加密 */
 export const hashPassword = (psw: string) => {
@@ -45,7 +47,7 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler {
    req: NextApiRequest,
    res: NextApiResponse
  ) {
-    const methods = ['GET', 'HEAD', 'PUT', 'PATCH', 'POST', 'DELETE'];
+    const methods = ['GET', 'eHEAD', 'PUT', 'PATCH', 'POST', 'DELETE'];
    const origin = req.headers.origin;
    await NextCors(req, res, {
      methods,
@@ -56,3 +58,15 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler {
    return handler(req, res);
  };
 }
+
+export const startQueue = () => {
+  const qaMax = Number(process.env.QA_MAX_PROCESS || 10);
+  const vectorMax = Number(process.env.VECTOR_MAX_PROCESS || 10);
+
+  for (let i = 0; i < qaMax; i++) {
+    generateQA();
+  }
+  for (let i = 0; i < vectorMax; i++) {
+    generateVector();
+  }
+};
--- a/src/store/global.ts
+++ b/src/store/global.ts
@@ -1,8 +1,12 @@
 import { create } from 'zustand';
 import { devtools } from 'zustand/middleware';
 import { immer } from 'zustand/middleware/immer';
+import type { InitDateResponse } from '@/pages/api/system/getInitData';
+import { getInitData } from '@/api/system';

 type State = {
+  initData: InitDateResponse;
+  loadInitData: () => Promise<void>;
  loading: boolean;
  setLoading: (val: boolean) => null;
  screenWidth: number;
@@ -13,6 +17,18 @@ type State = {
 export const useGlobalStore = create<State>()(
  devtools(
    immer((set, get) => ({
+      initData: {
+        beianText: '',
+        googleVerKey: ''
+      },
+      async loadInitData() {
+        try {
+          const res = await getInitData();
+          set((state) => {
+            state.initData = res;
+          });
+        } catch (error) {}
+      },
      loading: false,
      setLoading: (val: boolean) => {
        set((state) => {
--- a/src/styles/reset.scss
+++ b/src/styles/reset.scss
@@ -25,10 +25,6 @@ svg {
  margin: 0;
 }

-#__next {
-  height: 100%;
-}
-
 ::-webkit-scrollbar {
  width: 8px;
  height: 8px;
@@ -65,16 +61,26 @@ textarea::placeholder {
  font-size: 0.85em;
 }

+* {
+  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
+  -webkit-focus-ring-color: rgba(0, 0, 0, 0);
+  outline: none;
+}
+
+#__next {
+  height: 100%;
+}
+#nprogress .bar {
+  background: '#85b1ff' !important; //自定义颜色
+}
+
 .textEllipsis {
  text-overflow: ellipsis;
  white-space: nowrap;
  overflow: hidden;
 }
-
-* {
-  -webkit-tap-highlight-color: rgba(0, 0, 0, 0);
-  -webkit-focus-ring-color: rgba(0, 0, 0, 0);
-  outline: none;
+.grecaptcha-badge {
+  display: none !important;
 }

@media (max-width: 900px) {
@@ -93,7 +99,3 @@ textarea::placeholder {
    padding-bottom: env(safe-area-inset-bottom);
  }
 }
-
-#nprogress .bar {
-  background: '#85b1ff' !important; //自定义颜色
-}
--- a/src/types/index.d.ts
+++ b/src/types/index.d.ts
@@ -1,18 +1,16 @@
 import type { Mongoose } from 'mongoose';
-import type { RedisClientType } from 'redis';
 import type { Agent } from 'http';
 import type { Pool } from 'pg';

 declare global {
  var mongodb: Mongoose | string | null;
-  var redisClient: RedisClientType | null;
  var pgClient: Pool | null;
-  var generatingQA: boolean;
-  var generatingAbstract: boolean;
-  var generatingVector: boolean;
-  var QRCode: any;
  var httpsAgent: Agent;
  var particlesJS: any;
+  var grecaptcha: any;
+  var QRCode: any;
+  var qaQueueLen: number;
+  var vectorQueueLen: number;

  interface Window {
    ['pdfjs-dist/build/pdf']: any;
--- a/src/types/model.d.ts
+++ b/src/types/model.d.ts
@@ -1,6 +1,6 @@
 import { ModelStatusEnum } from '@/constants/model';
 import type { ModelSchema, kbSchema } from './mongoSchema';
-import { ChatModelType, ModelVectorSearchModeEnum } from '@/constants/model';
+import { ChatModelType, appVectorSearchModeEnum } from '@/constants/model';

 export type ModelListItemType = {
  _id: string;
--- a/src/types/mongoSchema.d.ts
+++ b/src/types/mongoSchema.d.ts
@@ -2,12 +2,13 @@ import type { ChatItemType } from './chat';
 import {
  ModelStatusEnum,
  ModelNameEnum,
-  ModelVectorSearchModeEnum,
+  appVectorSearchModeEnum,
  ChatModelType,
  EmbeddingModelType
 } from '@/constants/model';
 import type { DataType } from './data';
 import { BillTypeEnum } from '@/constants/user';
+import { TrainingModeEnum } from '@/constants/plugin';

 export interface UserModelSchema {
  _id: string;
@@ -44,7 +45,7 @@ export interface ModelSchema {
  updateTime: number;
  chat: {
    relatedKbs: string[];
-    searchMode: `${ModelVectorSearchModeEnum}`;
+    searchMode: `${appVectorSearchModeEnum}`;
    systemPrompt: string;
    temperature: number;
    chatModel: ChatModelType; // 聊天时用的模型，训练后就是训练的模型
@@ -68,13 +69,15 @@ export interface CollectionSchema {

 export type ModelDataType = 0 | 1;

-export interface SplitDataSchema {
+export interface TrainingDataSchema {
  _id: string;
  userId: string;
  kbId: string;
+  lockTime: Date;
+  mode: `${TrainingModeEnum}`;
  prompt: string;
-  errorText: string;
-  textList: string[];
+  q: string;
+  a: string;
 }

 export interface ChatSchema {
--- a/src/types/pg.d.ts
+++ b/src/types/pg.d.ts
@@ -1,11 +1,7 @@
-import { ModelDataStatusEnum } from '@/constants/model';
-
 export interface PgKBDataItemType {
  id: string;
  q: string;
  a: string;
-  status: `${ModelDataStatusEnum}`;
-  // model_id: string;
-  // user_id: string;
-  // kb_id: string;
+  user_id: string;
+  kb_id: string;
 }
--- a/src/types/plugin.d.ts
+++ b/src/types/plugin.d.ts
@@ -1,5 +1,4 @@
 import type { kbSchema } from './mongoSchema';
-import { PluginTypeEnum } from '@/constants/plugin';

 /* kb type */
 export interface KbItemType extends kbSchema {
@@ -9,27 +8,12 @@ export interface KbItemType extends kbSchema {

 export interface KbDataItemType {
  id: string;
-  status: 'waiting' | 'ready';
  q: string; // 提问词
  a: string; // 原文
  kbId: string;
  userId: string;
 }

-/* plugin */
-export interface PluginConfig {
-  name: string;
-  desc: string;
-  url: string;
-  category: `${PluginTypeEnum}`;
-  uniPrice: 22; // 1k token
-  params: [
-    {
-      type: '';
-    }
-  ];
-}
-
 export type TextPluginRequestParams = {
  input: string;
 };
--- a/src/utils/file.ts
+++ b/src/utils/file.ts
@@ -145,7 +145,7 @@ export const fileDownload = ({
 * slideLen - The size of the before and after Text
 * maxLen > slideLen
 */
-export const splitText_token = ({
+export const splitText_token = async ({
  text,
  maxLen,
  slideLen
@@ -154,32 +154,39 @@ export const splitText_token = ({
  maxLen: number;
  slideLen: number;
 }) => {
-  const enc = getOpenAiEncMap()['gpt-3.5-turbo'];
-  // filter empty text. encode sentence
-  const encodeText = enc.encode(text);
+  try {
+    const enc = getOpenAiEncMap()['gpt-3.5-turbo'];
+    // filter empty text. encode sentence
+    const encodeText = enc.encode(text);

-  const chunks: string[] = [];
-  let tokens = 0;
+    const chunks: string[] = [];
+    let tokens = 0;

-  let startIndex = 0;
-  let endIndex = Math.min(startIndex + maxLen, encodeText.length);
-  let chunkEncodeArr = encodeText.slice(startIndex, endIndex);
+    let startIndex = 0;
+    let endIndex = Math.min(startIndex + maxLen, encodeText.length);
+    let chunkEncodeArr = encodeText.slice(startIndex, endIndex);

-  const decoder = new TextDecoder();
+    const decoder = new TextDecoder();

-  while (startIndex < encodeText.length) {
-    tokens += chunkEncodeArr.length;
-    chunks.push(decoder.decode(enc.decode(chunkEncodeArr)));
+    while (startIndex < encodeText.length) {
+      tokens += chunkEncodeArr.length;
+      chunks.push(decoder.decode(enc.decode(chunkEncodeArr)));

-    startIndex += maxLen - slideLen;
-    endIndex = Math.min(startIndex + maxLen, encodeText.length);
-    chunkEncodeArr = encodeText.slice(Math.min(encodeText.length - slideLen, startIndex), endIndex);
+      startIndex += maxLen - slideLen;
+      endIndex = Math.min(startIndex + maxLen, encodeText.length);
+      chunkEncodeArr = encodeText.slice(
+        Math.min(encodeText.length - slideLen, startIndex),
+        endIndex
+      );
+    }
+
+    return {
+      chunks,
+      tokens
+    };
+  } catch (error) {
+    return Promise.reject(error);
  }
-
-  return {
-    chunks,
-    tokens
-  };
 };

 export const fileToBase64 = (file: File) => {
--- a/src/utils/plugin/google.ts
+++ b/src/utils/plugin/google.ts
@@ -0,0 +1,34 @@
+import axios from 'axios';
+import { Obj2Query } from '../tools';
+
+export const getClientToken = (googleVerKey: string) => {
+  if (typeof grecaptcha === 'undefined' || !grecaptcha?.ready) return '';
+  return new Promise<string>((resolve, reject) => {
+    grecaptcha.ready(async () => {
+      try {
+        const token = await grecaptcha.execute(googleVerKey, {
+          action: 'submit'
+        });
+        resolve(token);
+      } catch (error) {
+        reject(error);
+      }
+    });
+  });
+};
+
+// service run
+export const authGoogleToken = async (data: {
+  secret: string;
+  response: string;
+  remoteip?: string;
+}) => {
+  const res = await axios.post<{ score?: number; success: boolean; 'error-codes': string[] }>(
+    `https://www.recaptcha.net/recaptcha/api/siteverify?${Obj2Query(data)}`
+  );
+
+  if (res.data.success && res.data.score && res.data.score >= 0.7) {
+    return Promise.resolve('');
+  }
+  return Promise.reject(res.data['error-codes'][0] || '非法环境');
+};
Author	SHA1	Message	Date
archer	403e1f2d92	perf: export data	2023-05-28 21:13:17 +08:00
archer	d351a56e03	docs	2023-05-28 21:02:18 +08:00
archer	516618b0cd	feat: insert data de-weight;perf: input queue	2023-05-28 20:13:19 +08:00
archer	7e99f905bc	perf: random queue	2023-05-28 16:16:59 +08:00
archer	a287ace126	perf: code	2023-05-28 10:23:14 +08:00
archer	4f0bd677f2	docs	2023-05-27 15:10:19 +08:00
archer	741381ecb0	perf: generate queue	2023-05-27 15:05:30 +08:00
archer	f05b12975c	perf: google message	2023-05-27 00:15:03 +08:00
archer	85e94966ac	fix: kb delete and google auth	2023-05-26 23:37:21 +08:00
archer	dc1c1d1355	training queue	2023-05-26 23:08:25 +08:00
archer	69f32a0861	fix: search rate	2023-05-26 15:36:18 +08:00
archer	c99d6998ea	feat: google auth	2023-05-26 13:59:41 +08:00
HDS	116e9c8d85	fix: 修改了init.sql重复的bug (#62 )	2023-05-26 08:47:48 +08:00
archer	52920726d4	fix: default isEdit	2023-05-23 19:33:59 +08:00