Extraction schema (#398)

Dockerfile

@@ -9,8 +9,7 @@ ARG name
 # copy packages and one project
 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
 COPY ./packages ./packages
-COPY ./projects/$name/package.json ./projects/$name/package.json
+COPY ./projects/$name ./projects/$name
-COPY ./projects/$name/pnpm-lock.yaml ./projects/$name/pnpm-lock.yaml
 
 RUN \
   [ -f pnpm-lock.yaml ] && pnpm install || \
@@ -27,10 +26,7 @@ ARG name
 # copy common node_modules and one project node_modules
 COPY --from=deps /app/node_modules ./node_modules
 COPY --from=deps /app/packages ./packages
-COPY ./projects/$name ./projects/$name
+COPY --from=deps /app/projects/$name ./projects/$name
-COPY --from=deps /app/projects/$name/node_modules ./projects/$name/node_modules
-COPY pnpm-lock.yaml pnpm-workspace.yaml ./
-COPY ./packages ./packages
 
 # Uncomment the following line in case you want to disable telemetry during the build.
 ENV NEXT_TELEMETRY_DISABLED 1

docSite/content/docs/development/configuration.md

@@ -23,7 +23,7 @@ weight: 520
   "SystemParams": {
     "vectorMaxProcess": 15, // 向量生成最大进程，结合数据库性能和 key 来设置
     "qaMaxProcess": 15,  // QA 生成最大进程，结合数据库性能和 key 来设置
-    "pgIvfflatProbe": 20  // pg vector 搜索探针。没有设置索引前可忽略，通常 50w 组以上才需要设置。
+    "pgHNSWEfSearch": 40  // pg vector 索引参数，越大精度高但速度慢
   },
   "ChatModels": [
     {

docSite/content/docs/development/intro.md

@@ -66,7 +66,7 @@ git clone git@github.com:<github_username>/FastGPT.git
 
 - `vectorMaxProcess`: 向量生成最大进程，根据数据库和 key 的并发数来决定，通常单个 120 号，2c4g 服务器设置 10~15。
 - `qaMaxProcess`: QA 生成最大进程
-- `pgIvfflatProbe`: PostgreSQL vector 搜索探针，没有添加 vector 索引时可忽略。
+- `pgHNSWEfSearch`: PostgreSQL vector 索引参数，越大搜索精度越高但是速度越慢，具体可看 pgvector 官方说明。
 
 ### 5. 运行
 

docSite/content/docs/installation/docker.md

@@ -72,7 +72,7 @@ brew install orbstack
 
 依次执行下面命令，创建 FastGPT 文件并拉取`docker-compose.yml`和`config.json`，执行完后目录下会有 2 个文件。
 
-非 Linux 环境或无法访问外网环境，可手动创建一个目录，并下载下面2个链接的文件。
+非 Linux 环境或无法访问外网环境，可手动创建一个目录，并下载下面2个链接的文件: [docker-compose.yml](https://github.com/labring/FastGPT/blob/main/files/deploy/fastgpt/docker-compose.yml),[config.json](https://github.com/labring/FastGPT/blob/main/projects/app/data/config.json)
 
 **注意: `docker-compose.yml` 配置文件中 Mongo 为 5.x，部分服务器不支持，需手动更改其镜像版本为 4.4.24**
 

docSite/content/docs/pricing.md

@@ -30,6 +30,10 @@ weight: 10
 | 星火2.0 - 对话         | 0.01                              |
 | chatglm_pro - 对话     | 0.01                              |
 | 通义千问 - 对话         | 0.01                              |
+| 问题分类         | 0.03                             |
+| 内容提取         | 0.03                             |
+| 下一步指引         | 0.015                              |
+
 {{< /table >}}
 
 {{% alert context="warning" %}}

files/deploy/fastgpt/docker-compose.yml

@@ -2,8 +2,8 @@
 version: '3.3'
 services:
   pg:
-    image: ankane/pgvector:v0.4.2 # git
+    image: ankane/pgvector:v0.5.0 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.4.2 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.5.0 # 阿里云
     container_name: pg
     restart: always
     ports: # 生产环境建议不要暴露
@@ -66,8 +66,8 @@ networks:
 # version: '3.3'
 # services:
 #   pg:
-#   image: ankane/pgvector:v0.4.2 # dockerhub
+#   image: ankane/pgvector:v0.5.0 # dockerhub
-#   # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.4.2 # 阿里云
+#   # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/pgvector:v0.5.0 # 阿里云
 #   container_name: pg
 #   restart: always
 #   ports: # 生产环境建议不要暴露

package.json

@@ -8,11 +8,11 @@
   },
   "devDependencies": {
     "husky": "^8.0.3",
-    "i18next": "^23.2.11",
     "lint-staged": "^13.2.1",
-    "next-i18next": "^14.0.0",
     "prettier": "^3.0.3",
-    "react-i18next": "^13.0.2"
+    "i18next": "^23.2.11",
+    "react-i18next": "^13.0.2",
+    "next-i18next": "^14.0.0"
   },
   "lint-staged": {
     "./**/**/*.{ts,tsx,scss}": "npm run format"

packages/common/mongo/index.ts

@@ -0,0 +1,6 @@
+import mongoose from 'mongoose';
+
+export default mongoose;
+export * from 'mongoose';
+
+export const connectionMongo = global.mongodb || mongoose;

packages/common/mongo/init.ts

@@ -0,0 +1,74 @@
+import mongoose from './index';
+import 'winston-mongodb';
+import { createLogger, format, transports } from 'winston';
+
+/**
+ * connect MongoDB and init data
+ */
+export async function connectMongo({
+  beforeHook,
+  afterHook
+}: {
+  beforeHook?: () => any;
+  afterHook?: () => any;
+}): Promise<void> {
+  if (global.mongodb) {
+    return;
+  }
+  global.mongodb = mongoose;
+
+  beforeHook && (await beforeHook());
+
+  // logger
+  initLogger();
+
+  console.log('mongo start connect');
+  try {
+    mongoose.set('strictQuery', true);
+    await mongoose.connect(process.env.MONGODB_URI as string, {
+      bufferCommands: true,
+      maxConnecting: Number(process.env.DB_MAX_LINK || 5),
+      maxPoolSize: Number(process.env.DB_MAX_LINK || 5),
+      minPoolSize: 2
+    });
+
+    console.log('mongo connected');
+
+    afterHook && (await afterHook());
+  } catch (error) {
+    console.log('error->', 'mongo connect error', error);
+    global.mongodb = undefined;
+  }
+}
+
+function initLogger() {
+  global.logger = createLogger({
+    transports: [
+      new transports.MongoDB({
+        db: process.env.MONGODB_URI as string,
+        collection: 'server_logs',
+        options: {
+          useUnifiedTopology: true
+        },
+        cappedSize: 500000000,
+        tryReconnect: true,
+        metaKey: 'meta',
+        format: format.combine(format.timestamp(), format.json())
+      }),
+      new transports.Console({
+        format: format.combine(
+          format.timestamp({ format: 'YYYY-MM-DD HH:mm:ss' }),
+          format.printf((info) => {
+            if (info.level === 'error') {
+              console.log(info.meta);
+              return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}`;
+            }
+            return `[${info.level.toLocaleUpperCase()}]: ${[info.timestamp]}: ${info.message}${
+              info.meta ? `: ${JSON.stringify(info.meta)}` : ''
+            }`;
+          })
+        )
+      })
+    ]
+  });
+}

packages/common/package.json

@@ -1,4 +1,13 @@
 {
   "name": "@fastgpt/common",
-  "version": "1.0.0"
+  "version": "1.0.0",
+  "dependencies": {
+    "mongoose": "^7.0.2",
+    "winston": "^3.10.0",
+    "winston-mongodb": "^5.1.1",
+    "axios": "^1.5.1"
+  },
+  "devDependencies": {
+    "@types/node": "^20.8.5"
+  }
 }

packages/common/plusApi/censor.ts

@@ -1,7 +1,7 @@
 import { POST } from './request';
 
 export const postTextCensor = (data: { text: string }) =>
-  POST<{ code?: number; message: string }>('/plugins/censor/text_baidu', data)
+  POST<{ code?: number; message: string }>('/common/censor/text_baidu', data)
     .then((res) => {
       if (res?.code === 5000) {
         return Promise.reject(res);

packages/common/tools/str.ts

@@ -1,5 +1,32 @@
+import crypto from 'crypto';
+
 export function strIsLink(str?: string) {
   if (!str) return false;
   if (/^((http|https)?:\/\/|www\.|\/)[^\s/$.?#].[^\s]*$/i.test(str)) return true;
   return false;
 }
+
+export const hashStr = (psw: string) => {
+  return crypto.createHash('sha256').update(psw).digest('hex');
+};
+
+/* simple text, remove chinese space and extra \n */
+export const simpleText = (text: string) => {
+  text = text.replace(/([\u4e00-\u9fa5])[\s&&[^\n]]+([\u4e00-\u9fa5])/g, '$1$2');
+  text = text.replace(/\n{2,}/g, '\n');
+  text = text.replace(/[\s&&[^\n]]{2,}/g, ' ');
+  text = text.replace(/[\x00-\x08]/g, ' ');
+
+  // replace empty \n
+  let newText = '';
+  let lastChar = '';
+  for (let i = 0; i < text.length; i++) {
+    const currentChar = text[i];
+    if (currentChar === '\n' && !/[。？！；.?!;]/g.test(lastChar)) {
+    } else {
+      newText += currentChar;
+    }
+    lastChar = currentChar;
+  }
+  return newText;
+};

packages/common/type/index.d.ts

@@ -0,0 +1,43 @@
+import type { Mongoose } from '../mongo';
+import type { Logger } from 'winston';
+
+export type FeConfigsType = {
+  show_emptyChat?: boolean;
+  show_register?: boolean;
+  show_appStore?: boolean;
+  show_contact?: boolean;
+  show_git?: boolean;
+  show_doc?: boolean;
+  show_pay?: boolean;
+  show_openai_account?: boolean;
+  show_promotion?: boolean;
+  hide_app_flow?: boolean;
+  openAPIUrl?: string;
+  systemTitle?: string;
+  authorText?: string;
+  googleClientVerKey?: string;
+  isPlus?: boolean;
+  oauth?: {
+    github?: string;
+    google?: string;
+  };
+  limit?: {
+    exportLimitMinutes?: number;
+  };
+  scripts?: { [key: string]: string }[];
+};
+
+export type SystemEnvType = {
+  pluginBaseUrl?: string;
+  openapiPrefix?: string;
+  vectorMaxProcess: number;
+  qaMaxProcess: number;
+  pgHNSWEfSearch: number;
+};
+
+declare global {
+  var mongodb: Mongoose | undefined;
+  var logger: Logger;
+  var feConfigs: FeConfigsType;
+  var systemEnv: SystemEnvType;
+}

packages/core/ai/config.ts

@@ -1,4 +1,4 @@
-import { UserModelSchema } from '../user/type';
+import type { UserModelSchema } from '@fastgpt/support/user/type.d';
 import OpenAI from 'openai';
 
 export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';

packages/core/ai/functions/createQuestionGuide.ts

@@ -10,7 +10,7 @@ export async function createQuestionGuide({
   messages: ChatCompletionRequestMessage[];
   model: string;
 }) {
-  const ai = getAIApi();
+  const ai = getAIApi(undefined, 48000);
   const data = await ai.chat.completions.create({
     model: model,
     temperature: 0,
@@ -25,7 +25,7 @@ export async function createQuestionGuide({
     stream: false
   });
 
-  const answer = data.choices?.[0].message?.content || '';
+  const answer = data.choices?.[0]?.message?.content || '';
   const totalTokens = data.usage?.total_tokens || 0;
 
   const start = answer.indexOf('[');

packages/core/dataset/constant.ts

@@ -1,3 +1,22 @@
+export enum DatasetTypeEnum {
+  folder = 'folder',
+  dataset = 'dataset'
+}
+
+export const DatasetTypeMap = {
+  [DatasetTypeEnum.folder]: {
+    name: 'folder'
+  },
+  [DatasetTypeEnum.dataset]: {
+    name: 'dataset'
+  }
+};
+
+export enum FileStatusEnum {
+  embedding = 'embedding',
+  ready = 'ready'
+}
+
 export enum DatasetSpecialIdEnum {
   manual = 'manual',
   mark = 'mark'
@@ -13,3 +32,5 @@ export const datasetSpecialIdMap = {
   }
 };
 export const datasetSpecialIds: string[] = [DatasetSpecialIdEnum.manual, DatasetSpecialIdEnum.mark];
+
+export const FolderAvatarSrc = '/imgs/files/folder.svg';

packages/core/dataset/schema.ts

@@ -1,8 +1,9 @@
-import { Schema, model, models, Model } from 'mongoose';
+import { connectionMongo, type Model } from '@fastgpt/common/mongo';
-import { kbSchema as SchemaType } from '@/types/mongoSchema';
+const { Schema, model, models } = connectionMongo;
-import { KbTypeMap } from '@/constants/dataset';
+import { DatasetSchemaType } from './type';
+import { DatasetTypeMap } from './constant';
 
-const kbSchema = new Schema({
+const DatasetSchema = new Schema({
   parentId: {
     type: Schema.Types.ObjectId,
     ref: 'kb',
@@ -32,7 +33,7 @@ const kbSchema = new Schema({
   },
   type: {
     type: String,
-    enum: Object.keys(KbTypeMap),
+    enum: Object.keys(DatasetTypeMap),
     required: true,
     default: 'dataset'
   },
@@ -42,4 +43,4 @@ const kbSchema = new Schema({
   }
 });
 
-export const KB: Model<SchemaType> = models['kb'] || model('kb', kbSchema);
+export const MongoDataset: Model<DatasetSchemaType> = models['kb'] || model('kb', DatasetSchema);

packages/core/dataset/type.d.ts

@@ -0,0 +1,13 @@
+import { DatasetTypeEnum } from './constant';
+
+export type DatasetSchemaType = {
+  _id: string;
+  userId: string;
+  parentId: string;
+  updateTime: Date;
+  avatar: string;
+  name: string;
+  vectorModel: string;
+  tags: string[];
+  type: `${DatasetTypeEnum}`;
+};

packages/core/tsconfig.json

@@ -16,6 +16,6 @@
     "incremental": true,
     "baseUrl": "."
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts"],
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"],
   "exclude": ["node_modules"]
 }

packages/support/openapi/auth.ts

@@ -0,0 +1,32 @@
+import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
+import { updateApiKeyUsedTime } from './tools';
+import { MongoOpenApi } from './schema';
+import { POST } from '@fastgpt/common/plusApi/request';
+import { OpenApiSchema } from './type.d';
+
+export type AuthOpenApiLimitProps = { openApi: OpenApiSchema };
+
+export async function authOpenApiKey({ apikey }: { apikey: string }) {
+  if (!apikey) {
+    return Promise.reject(ERROR_ENUM.unAuthApiKey);
+  }
+
+  try {
+    const openApi = await MongoOpenApi.findOne({ apiKey: apikey });
+    if (!openApi) {
+      return Promise.reject(ERROR_ENUM.unAuthApiKey);
+    }
+    const userId = String(openApi.userId);
+
+    // auth limit
+    if (global.feConfigs?.isPlus) {
+      await POST('/support/openapi/authLimit', { openApi } as AuthOpenApiLimitProps);
+    }
+
+    updateApiKeyUsedTime(openApi._id);
+
+    return { apikey, userId, appId: openApi.appId };
+  } catch (error) {
+    return Promise.reject(error);
+  }
+}

packages/support/openapi/schema.ts

@@ -1,5 +1,6 @@
-import { Schema, model, models, Model } from 'mongoose';
+import { connectionMongo, type Model } from '@fastgpt/common/mongo';
-import { OpenApiSchema } from '@/types/support/openapi';
+const { Schema, model, models } = connectionMongo;
+import { OpenApiSchema } from './type.d';
 import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
 import { formatPrice } from '@fastgpt/common/bill/index';
 
@@ -54,4 +55,5 @@ const OpenApiSchema = new Schema(
   }
 );
 
-export const OpenApi: Model<OpenApiSchema> = models['openapi'] || model('openapi', OpenApiSchema);
+export const MongoOpenApi: Model<OpenApiSchema> =
+  models['openapi'] || model('openapi', OpenApiSchema);

packages/support/openapi/tools.ts

@@ -1,13 +1,13 @@
-import { OpenApi } from './schema';
+import { MongoOpenApi } from './schema';
 
 export async function updateApiKeyUsedTime(id: string) {
-  await OpenApi.findByIdAndUpdate(id, {
+  await MongoOpenApi.findByIdAndUpdate(id, {
     lastUsedTime: new Date()
   });
 }
 
 export async function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) {
-  await OpenApi.findOneAndUpdate(
+  await MongoOpenApi.findOneAndUpdate(
     { apiKey: apikey },
     {
       $inc: {

packages/support/outLink/auth.ts

@@ -0,0 +1,67 @@
+import { AuthUserTypeEnum, authBalanceByUid } from '../user/auth';
+import { MongoOutLink } from './schema';
+import { POST } from '@fastgpt/common/plusApi/request';
+import { OutLinkSchema } from './type.d';
+
+export type AuthLinkProps = { ip?: string | null; authToken?: string; question: string };
+export type AuthLinkLimitProps = AuthLinkProps & { outLink: OutLinkSchema };
+
+export async function authOutLinkChat({
+  shareId,
+  ip,
+  authToken,
+  question
+}: AuthLinkProps & {
+  shareId: string;
+}) {
+  // get outLink
+  const outLink = await MongoOutLink.findOne({
+    shareId
+  });
+
+  if (!outLink) {
+    return Promise.reject('分享链接无效');
+  }
+
+  const uid = String(outLink.userId);
+
+  const [user] = await Promise.all([
+    authBalanceByUid(uid), // authBalance
+    ...(global.feConfigs?.isPlus ? [authOutLinkLimit({ outLink, ip, authToken, question })] : []) // limit auth
+  ]);
+
+  return {
+    user,
+    userId: String(outLink.userId),
+    appId: String(outLink.appId),
+    authType: AuthUserTypeEnum.token,
+    responseDetail: outLink.responseDetail
+  };
+}
+
+export function authOutLinkLimit(data: AuthLinkLimitProps) {
+  return POST('/support/outLink/authLimit', data);
+}
+
+export async function authOutLinkId({ id }: { id: string }) {
+  const outLink = await MongoOutLink.findOne({
+    shareId: id
+  });
+
+  if (!outLink) {
+    return Promise.reject('分享链接无效');
+  }
+
+  return {
+    userId: String(outLink.userId)
+  };
+}
+
+export type AuthShareChatInitProps = {
+  authToken?: string;
+  tokenUrl?: string;
+};
+
+export function authShareChatInit(data: AuthShareChatInitProps) {
+  return POST('/support/outLink/authShareChatInit', data);
+}

packages/support/outLink/constant.ts

@@ -0,0 +1,5 @@
+export enum OutLinkTypeEnum {
+  'share' = 'share',
+  'iframe' = 'iframe',
+  apikey = 'apikey'
+}

packages/support/outLink/schema.ts

@@ -1,6 +1,7 @@
-import { Schema, model, models, Model } from 'mongoose';
+import { connectionMongo, type Model } from '@fastgpt/common/mongo';
-import { OutLinkSchema as SchemaType } from '@/types/support/outLink';
+const { Schema, model, models } = connectionMongo;
-import { OutLinkTypeEnum } from '@/constants/chat';
+import { OutLinkSchema as SchemaType } from './type.d';
+import { OutLinkTypeEnum } from './constant';
 
 const OutLinkSchema = new Schema({
   shareId: {
@@ -55,4 +56,5 @@ const OutLinkSchema = new Schema({
   }
 });
 
-export const OutLink: Model<SchemaType> = models['outlinks'] || model('outlinks', OutLinkSchema);
+export const MongoOutLink: Model<SchemaType> =
+  models['outlinks'] || model('outlinks', OutLinkSchema);

packages/support/outLink/tools.ts

@@ -1,7 +1,5 @@
-import { addLog } from '@/service/utils/tools';
-import { ChatHistoryItemResType } from '@/types/chat';
 import axios from 'axios';
-import { OutLink } from './schema';
+import { MongoOutLink } from './schema';
 
 export const updateOutLinkUsage = async ({
   shareId,
@@ -11,7 +9,7 @@ export const updateOutLinkUsage = async ({
   total: number;
 }) => {
   try {
-    await OutLink.findOneAndUpdate(
+    await MongoOutLink.findOneAndUpdate(
       { shareId },
       {
         $inc: { total },
@@ -19,7 +17,7 @@ export const updateOutLinkUsage = async ({
       }
     );
   } catch (err) {
-    addLog.error('update shareChat error', err);
+    console.log('update shareChat error', err);
   }
 };
 
@@ -30,11 +28,11 @@ export const pushResult2Remote = async ({
 }: {
   authToken?: string;
   shareId?: string;
-  responseData?: ChatHistoryItemResType[];
+  responseData?: any[];
 }) => {
   if (!shareId || !authToken) return;
   try {
-    const outLink = await OutLink.findOne({
+    const outLink = await MongoOutLink.findOne({
       shareId
     });
     if (!outLink?.limit?.hookUrl) return;

packages/support/outLink/type.d.ts

@@ -1,6 +1,6 @@
-import { OutLinkTypeEnum } from '@/constants/chat';
+import { OutLinkTypeEnum } from './constant';
 
-export interface OutLinkSchema {
+export type OutLinkSchema = {
   _id: string;
   shareId: string;
   userId: string;
@@ -16,7 +16,7 @@ export interface OutLinkSchema {
     credit: number;
     hookUrl?: string;
   };
-}
+};
 
 export type OutLinkEditType = {
   _id?: string;

packages/support/package.json

@@ -2,6 +2,13 @@
   "name": "@fastgpt/support",
   "version": "1.0.0",
   "dependencies": {
-    "@fastgpt/common": "workspace:*"
+    "@fastgpt/common": "workspace:*",
+    "cookie": "^0.5.0",
+    "jsonwebtoken": "^9.0.2",
+    "axios": "^1.5.1"
+  },
+  "devDependencies": {
+    "@types/cookie": "^0.5.2",
+    "@types/jsonwebtoken": "^9.0.3"
   }
 }

packages/support/tsconfig.json

@@ -14,8 +14,11 @@
     "isolatedModules": true,
     "jsx": "preserve",
     "incremental": true,
-    "baseUrl": "."
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["./*"]
+    }
   },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts"],
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"],
   "exclude": ["node_modules"]
 }

packages/support/user/auth.ts

@@ -0,0 +1,167 @@
+import Cookie from 'cookie';
+import { authJWT } from './tools';
+import { authOpenApiKey } from '../openapi/auth';
+import { authOutLinkId } from '../outLink/auth';
+
+import { MongoUser } from './schema';
+import type { UserModelSchema } from './type.d';
+import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
+
+export enum AuthUserTypeEnum {
+  token = 'token',
+  root = 'root',
+  apikey = 'apikey',
+  outLink = 'outLink'
+}
+
+/* auth balance */
+export const authBalanceByUid = async (uid: string) => {
+  const user = await MongoUser.findById<UserModelSchema>(
+    uid,
+    '_id username balance openaiAccount timezone'
+  );
+  if (!user) {
+    return Promise.reject(ERROR_ENUM.unAuthorization);
+  }
+
+  if (user.balance <= 0) {
+    return Promise.reject(ERROR_ENUM.insufficientQuota);
+  }
+  return user;
+};
+
+/* uniform auth user */
+export const authUser = async ({
+  req,
+  authToken = false,
+  authRoot = false,
+  authApiKey = false,
+  authBalance = false,
+  authOutLink
+}: {
+  req: any;
+  authToken?: boolean;
+  authRoot?: boolean;
+  authApiKey?: boolean;
+  authBalance?: boolean;
+  authOutLink?: boolean;
+}) => {
+  const authCookieToken = async (cookie?: string, token?: string): Promise<string> => {
+    // 获取 cookie
+    const cookies = Cookie.parse(cookie || '');
+    const cookieToken = cookies.token || token;
+
+    if (!cookieToken) {
+      return Promise.reject(ERROR_ENUM.unAuthorization);
+    }
+
+    return await authJWT(cookieToken);
+  };
+  // from authorization get apikey
+  const parseAuthorization = async (authorization?: string) => {
+    if (!authorization) {
+      return Promise.reject(ERROR_ENUM.unAuthorization);
+    }
+
+    // Bearer fastgpt-xxxx-appId
+    const auth = authorization.split(' ')[1];
+    if (!auth) {
+      return Promise.reject(ERROR_ENUM.unAuthorization);
+    }
+
+    const { apikey, appId: authorizationAppid = '' } = await (async () => {
+      const arr = auth.split('-');
+      // abandon
+      if (arr.length === 3) {
+        return {
+          apikey: `${arr[0]}-${arr[1]}`,
+          appId: arr[2]
+        };
+      }
+      if (arr.length === 2) {
+        return {
+          apikey: auth
+        };
+      }
+      return Promise.reject(ERROR_ENUM.unAuthorization);
+    })();
+
+    // auth apikey
+    const { userId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
+
+    return {
+      uid: userId,
+      apikey,
+      appId: apiKeyAppId || authorizationAppid
+    };
+  };
+  // root user
+  const parseRootKey = async (rootKey?: string, userId = '') => {
+    if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
+      return Promise.reject(ERROR_ENUM.unAuthorization);
+    }
+    return userId;
+  };
+
+  const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers || {}) as {
+    cookie?: string;
+    token?: string;
+    apikey?: string;
+    rootkey?: string; // abandon
+    userid?: string;
+    authorization?: string;
+  };
+  const { shareId } = (req?.body || {}) as { shareId?: string };
+
+  let uid = '';
+  let appId = '';
+  let openApiKey = apikey;
+  let authType: `${AuthUserTypeEnum}` = AuthUserTypeEnum.token;
+
+  if (authOutLink && shareId) {
+    const res = await authOutLinkId({ id: shareId });
+    uid = res.userId;
+    authType = AuthUserTypeEnum.outLink;
+  } else if (authToken && (cookie || token)) {
+    // user token(from fastgpt web)
+    uid = await authCookieToken(cookie, token);
+    authType = AuthUserTypeEnum.token;
+  } else if (authRoot && rootkey) {
+    // root user
+    uid = await parseRootKey(rootkey, userid);
+    authType = AuthUserTypeEnum.root;
+  } else if (authApiKey && apikey) {
+    // apikey
+    const parseResult = await authOpenApiKey({ apikey });
+    uid = parseResult.userId;
+    authType = AuthUserTypeEnum.apikey;
+    openApiKey = parseResult.apikey;
+  } else if (authApiKey && authorization) {
+    // apikey from authorization
+    const authResponse = await parseAuthorization(authorization);
+    uid = authResponse.uid;
+    appId = authResponse.appId;
+    openApiKey = authResponse.apikey;
+    authType = AuthUserTypeEnum.apikey;
+  }
+
+  // not rootUser and no uid, reject request
+  if (!rootkey && !uid) {
+    return Promise.reject(ERROR_ENUM.unAuthorization);
+  }
+
+  // balance check
+  const user = await (() => {
+    if (authBalance) {
+      return authBalanceByUid(uid);
+    }
+  })();
+
+  return {
+    userId: String(uid),
+    appId,
+    authType,
+    user,
+    apikey: openApiKey
+  };
+};

packages/support/user/schema.ts

@@ -1,7 +1,8 @@
-import { Schema, model, models, Model } from 'mongoose';
+import { connectionMongo, type Model } from '@fastgpt/common/mongo';
-import { hashPassword } from '@/service/utils/tools';
+const { Schema, model, models } = connectionMongo;
+import { hashStr } from '@fastgpt/common/tools/str';
 import { PRICE_SCALE } from '@fastgpt/common/bill/constants';
-import { UserModelSchema } from '@/types/mongoSchema';
+import type { UserModelSchema } from './type.d';
 
 const UserSchema = new Schema({
   username: {
@@ -13,8 +14,8 @@ const UserSchema = new Schema({
   password: {
     type: String,
     required: true,
-    set: (val: string) => hashPassword(val),
+    set: (val: string) => hashStr(val),
-    get: (val: string) => hashPassword(val),
+    get: (val: string) => hashStr(val),
     select: false
   },
   createTime: {
@@ -42,6 +43,9 @@ const UserSchema = new Schema({
     exportKbTime: {
       // Every half hour
       type: Date
+    },
+    datasetMaxCount: {
+      type: Number
     }
   },
   openaiAccount: {
@@ -56,4 +60,4 @@ const UserSchema = new Schema({
   }
 });
 
-export const User: Model<UserModelSchema> = models['user'] || model('user', UserSchema);
+export const MongoUser: Model<UserModelSchema> = models['user'] || model('user', UserSchema);

packages/support/user/tools.ts

@@ -0,0 +1,28 @@
+import jwt from 'jsonwebtoken';
+import { ERROR_ENUM } from '@fastgpt/common/constant/errorCode';
+
+/* 生成 token */
+export const generateToken = (userId: string) => {
+  const key = process.env.TOKEN_KEY as string;
+  const token = jwt.sign(
+    {
+      userId,
+      exp: Math.floor(Date.now() / 1000) + 60 * 60 * 24 * 7
+    },
+    key
+  );
+  return token;
+};
+// auth token
+export const authJWT = (token: string) =>
+  new Promise<string>((resolve, reject) => {
+    const key = process.env.TOKEN_KEY as string;
+
+    jwt.verify(token, key, function (err, decoded: any) {
+      if (err || !decoded?.userId) {
+        reject(ERROR_ENUM.unAuthorization);
+        return;
+      }
+      resolve(decoded.userId);
+    });
+  });

packages/support/user/type.d.ts

@@ -15,5 +15,6 @@ export type UserModelSchema = {
   };
   limit: {
     exportKbTime?: Date;
+    datasetMaxCount?: number;
   };
 };

projects/app/data/config.json

@@ -2,7 +2,7 @@
   "SystemParams": {
     "vectorMaxProcess": 15,
     "qaMaxProcess": 15,
-    "pgIvfflatProbe": 20
+    "pgHNSWEfSearch ": 40
   },
   "ChatModels": [
     {

projects/app/next.config.js

@@ -5,9 +5,8 @@ const path = require('path');
 const nextConfig = {
   i18n,
   output: 'standalone',
-  reactStrictMode: false,
+  reactStrictMode: process.env.NODE_ENV === 'development' ? false : true,
   compress: true,
-  transpilePackages: ['@fastgpt/*'],
   webpack(config, { isServer }) {
     if (!isServer) {
       config.resolve = {
@@ -18,6 +17,16 @@ const nextConfig = {
         }
       };
     }
+    Object.assign(config.resolve.alias, {
+      '@mongodb-js/zstd': false,
+      '@aws-sdk/credential-providers': false,
+      snappy: false,
+      aws4: false,
+      'mongodb-client-encryption': false,
+      kerberos: false,
+      'supports-color': false,
+      'bson-ext': false
+    });
     config.module = {
       ...config.module,
       rules: config.module.rules.concat([
@@ -27,12 +36,15 @@ const nextConfig = {
           use: ['@svgr/webpack']
         }
       ]),
-      exprContextCritical: false
+      exprContextCritical: false,
+      unknownContextCritical: false
     };
 
     return config;
   },
+  transpilePackages: ['@fastgpt/*'],
   experimental: {
+    serverComponentsExternalPackages: ['mongoose'],
     outputFileTracingRoot: path.join(__dirname, '../../')
   }
 };

projects/app/package.json

@@ -22,9 +22,7 @@
     "@mozilla/readability": "^0.4.4",
     "@tanstack/react-query": "^4.24.10",
     "@types/nprogress": "^0.2.0",
-    "axios": "^1.3.3",
+    "axios": "^1.5.1",
-    "cookie": "^0.5.0",
-    "crypto": "^1.0.1",
     "date-fns": "^2.30.0",
     "dayjs": "^1.11.7",
     "downloadjs": "^1.4.7",
@@ -33,21 +31,18 @@
     "formidable": "^2.1.1",
     "framer-motion": "^9.0.6",
     "hyperdown": "^2.4.29",
-    "i18next": "^22.5.1",
     "immer": "^9.0.19",
     "js-cookie": "^3.0.5",
     "js-tiktoken": "^1.0.7",
     "jschardet": "^3.0.0",
     "jsdom": "^22.1.0",
-    "jsonwebtoken": "^9.0.0",
+    "jsonwebtoken": "^9.0.2",
     "lodash": "^4.17.21",
     "mammoth": "^1.5.1",
     "mermaid": "^10.2.3",
-    "mongoose": "^6.10.0",
     "multer": "1.4.5-lts.1",
     "nanoid": "^4.0.1",
     "next": "13.5.2",
-    "next-i18next": "^14.0.3",
     "nextjs-cors": "^2.1.2",
     "nprogress": "^0.2.0",
     "papaparse": "^5.4.1",
@@ -57,7 +52,6 @@
     "react-day-picker": "^8.7.1",
     "react-dom": "18.2.0",
     "react-hook-form": "^7.43.1",
-    "react-i18next": "^12.3.1",
     "react-markdown": "^8.0.7",
     "react-syntax-highlighter": "^15.5.0",
     "reactflow": "^11.7.4",
@@ -68,21 +62,20 @@
     "request-ip": "^3.3.0",
     "sass": "^1.58.3",
     "timezones-list": "^3.0.2",
-    "winston": "^3.10.0",
+    "zustand": "^4.3.5",
-    "winston-mongodb": "^5.1.1",
+    "i18next": "^23.2.11",
-    "zustand": "^4.3.5"
+    "react-i18next": "^13.0.2",
+    "next-i18next": "^14.0.0"
   },
   "devDependencies": {
     "@svgr/webpack": "^6.5.1",
-    "@types/cookie": "^0.5.1",
     "@types/downloadjs": "^1.4.3",
     "@types/formidable": "^2.0.5",
     "@types/js-cookie": "^3.0.3",
     "@types/jsdom": "^21.1.1",
-    "@types/jsonwebtoken": "^9.0.1",
+    "@types/jsonwebtoken": "^9.0.3",
     "@types/lodash": "^4.14.191",
     "@types/multer": "^1.4.7",
-    "@types/node": "18.14.0",
     "@types/papaparse": "^5.3.7",
     "@types/pg": "^8.6.6",
     "@types/react": "18.0.28",

projects/app/public/locales/en/common.json

@@ -181,6 +181,7 @@
     "Choice QA Desc": "The index is constructed with the structure of QA pairs, and ADAPTS to various scenarios such as Q&A and reading",
     "Choice Visual": "Visual workflow",
     "Choice Visual Desc": "Visualize modular operations, easily implement complex workflows, and make your AI no longer monolithic",
+    "Commercial": "Commercial",
     "Community": "Community",
     "Dateset": "",
     "Dateset Desc": "",

projects/app/public/locales/zh/common.json

@@ -181,6 +181,7 @@
     "Choice QA Desc": "采用 QA 对的结构构建索引，适应问答、阅读等多种场景",
     "Choice Visual": "可视化工作流",
     "Choice Visual Desc": "可视化模块操作，轻松实现复杂工作流，让你的 AI 不再单一",
+    "Commercial": "商务咨询",
     "Community": "社区",
     "Dateset": "自动数据预处理",
     "Dateset Desc": "提供手动输入、直接分段、LLM 自动处理和 CSV 等多种数据导入途径",

projects/app/src/components/ChatBox/SelectDataset.tsx

@@ -4,7 +4,7 @@ import { useTranslation } from 'next-i18next';
 import { useToast } from '@/web/common/hooks/useToast';
 import Avatar from '../Avatar';
 import MyIcon from '@/components/Icon';
-import { KbTypeEnum } from '@/constants/dataset';
+import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
 import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
 
 const SelectDataset = ({
@@ -57,7 +57,7 @@ const SelectDataset = ({
                       }
                     : {})}
                   onClick={() => {
-                    if (item.type === KbTypeEnum.folder) {
+                    if (item.type === DatasetTypeEnum.folder) {
                       setParentId(item._id);
                     } else {
                       setSelectedId(item._id);

projects/app/src/components/MyModal/index.tsx

@@ -45,8 +45,8 @@ const MyModal = ({
         {...props}
       >
         {!!title && <ModalHeader>{title}</ModalHeader>}
+        {onClose && <ModalCloseButton />}
         <Box overflow={'overlay'} h={'100%'}>
-          {onClose && <ModalCloseButton />}
           {children}
         </Box>
       </ModalContent>

projects/app/src/constants/chat.ts

@@ -53,11 +53,5 @@ export const ChatSourceMap = {
   }
 };
 
-export enum OutLinkTypeEnum {
-  'share' = 'share',
-  'iframe' = 'iframe',
-  apikey = 'apikey'
-}
-
 export const HUMAN_ICON = `/icon/human.svg`;
 export const LOGO_ICON = `/icon/logo.svg`;

projects/app/src/constants/dataset.ts

@@ -14,23 +14,3 @@ export const defaultKbDetail: DatasetItemType = {
     maxToken: 3000
   }
 };
-
-export enum KbTypeEnum {
-  folder = 'folder',
-  dataset = 'dataset'
-}
-export enum FileStatusEnum {
-  embedding = 'embedding',
-  ready = 'ready'
-}
-
-export const KbTypeMap = {
-  [KbTypeEnum.folder]: {
-    name: 'folder'
-  },
-  [KbTypeEnum.dataset]: {
-    name: 'dataset'
-  }
-};
-
-export const FolderAvatarSrc = '/imgs/files/folder.svg';

projects/app/src/constants/flow/index.ts

@@ -2,7 +2,7 @@ import type { BoxProps } from '@chakra-ui/react';
 
 export enum FlowInputItemTypeEnum {
   systemInput = 'systemInput', // history, userChatInput, variableInput
-  input = 'input',
+  input = 'input', // one line input
   textarea = 'textarea',
   numberInput = 'numberInput',
   select = 'select',
@@ -10,6 +10,7 @@ export enum FlowInputItemTypeEnum {
   custom = 'custom',
   target = 'target', // data input
   switch = 'switch',
+  chatInput = 'chatInput',
   selectApp = 'selectApp',
   // chat special input
   quoteList = 'quoteList',

projects/app/src/constants/model.ts

@@ -1,5 +1,5 @@
 import type { AppSchema } from '@/types/mongoSchema';
-import type { OutLinkEditType } from '@/types/support/outLink';
+import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
 
 export const defaultApp: AppSchema = {
   _id: '',

projects/app/src/global/common/api/systemRes.d.ts

@@ -4,11 +4,13 @@ import {
   type VectorModelItemType,
   FunctionModelItemType
 } from '@/types/model';
+import type { FeConfigsType } from '@fastgpt/common/type/index.d';
 
 export type InitDateResponse = {
   chatModels: ChatModelItemType[];
   qaModel: QAModelItemType;
   vectorModels: VectorModelItemType[];
   feConfigs: FeConfigsType;
+  priceMd: string;
   systemVersion: string;
 };

projects/app/src/global/core/api/datasetReq.d.ts

@@ -1,4 +1,4 @@
-import { KbTypeEnum } from '@/constants/dataset';
+import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
 import type { RequestPaging } from '@/types';
 import { TrainingModeEnum } from '@/constants/plugin';
 import type { SearchTestItemType } from '@/types/core/dataset';
@@ -18,7 +18,7 @@ export type CreateDatasetParams = {
   tags: string[];
   avatar: string;
   vectorModel?: string;
-  type: `${KbTypeEnum}`;
+  type: `${DatasetTypeEnum}`;
 };
 
 export type SearchTestProps = {

projects/app/src/global/core/api/datasetRes.d.ts

@@ -1,4 +1,3 @@
-import { KbTypeEnum } from '@/constants/dataset';
 import type { RequestPaging } from '@/types';
 import { TrainingModeEnum } from '@/constants/plugin';
 import type { SearchTestItemType } from '@/types/core/dataset';

projects/app/src/global/support/api/openapiReq.d.ts

@@ -1,4 +1,4 @@
-import type { OpenApiSchema } from '@/types/support/openapi';
+import type { OpenApiSchema } from '@fastgpt/support/openapi/type.d';
 
 export type GetApiKeyProps = {
   appId?: string;

projects/app/src/pages/_app.tsx

@@ -13,10 +13,10 @@ import { appWithTranslation, useTranslation } from 'next-i18next';
 import { getLangStore, setLangStore } from '@/web/common/utils/i18n';
 import { useRouter } from 'next/router';
 import { useGlobalStore } from '@/web/common/store/global';
+import type { FeConfigsType } from '@fastgpt/common/type/index.d';
 
 import 'nprogress/nprogress.css';
 import '@/web/styles/reset.scss';
-import { FeConfigsType } from '@/types';
 
 //Binding events.
 Router.events.on('routeChangeStart', () => NProgress.start());

projects/app/src/pages/_error.tsx

@@ -2,8 +2,6 @@ import { useEffect } from 'react';
 import { useRouter } from 'next/router';
 import { serviceSideProps } from '@/web/common/utils/i18n';
 import { useGlobalStore } from '@/web/common/store/global';
-import { addLog } from '@/service/utils/tools';
-import { getErrText } from '@/utils/tools';
 
 function Error() {
   const router = useRouter();
@@ -36,8 +34,6 @@ function Error() {
 export async function getServerSideProps(context: any) {
   console.log('[render error]: ', context);
 
-  addLog.error(getErrText(context?.res));
-
   return {
     props: { ...(await serviceSideProps(context)) }
   };

projects/app/src/pages/account/components/PayModal.tsx

@@ -6,10 +6,9 @@ import { useQuery } from '@tanstack/react-query';
 import { useRouter } from 'next/router';
 import { getErrText } from '@/utils/tools';
 import { useTranslation } from 'react-i18next';
-import { formatPrice } from '@fastgpt/common/bill/index';
 import Markdown from '@/components/Markdown';
 import MyModal from '@/components/MyModal';
-import { vectorModelList, chatModelList, qaModel } from '@/web/common/store/static';
+import { priceMd } from '@/web/common/store/static';
 
 const PayModal = ({ onClose }: { onClose: () => void }) => {
   const router = useRouter();
@@ -70,10 +69,15 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
       title={t('user.Pay')}
       isCentered={!payId}
     >
-      <ModalBody py={0}>
+      <ModalBody
+        p={0}
+        h={payId ? 'auto' : ['auto', '70vh']}
+        display={'flex'}
+        flexDirection={'column'}
+      >
         {!payId && (
           <>
-            <Grid gridTemplateColumns={'repeat(4,1fr)'} gridGap={5} mb={4}>
+            <Grid gridTemplateColumns={'repeat(4,1fr)'} gridGap={5} mb={4} px={6}>
               {[10, 20, 50, 100].map((item) => (
                 <Button
                   key={item}
@@ -84,7 +88,7 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
                 </Button>
               ))}
             </Grid>
-            <Box mb={4}>
+            <Box mb={4} px={6}>
               <Input
                 value={inputVal}
                 type={'number'}
@@ -95,18 +99,9 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
                 }}
               ></Input>
             </Box>
-            <Markdown
+            <Box flex={[1, '1 0 0']} overflow={'overlay'} px={6}>
-              source={`
+              <Markdown source={priceMd} />
-| 计费项 | 价格: 元/ 1K tokens(包含上下文)|
+            </Box>
-| --- | --- |
-${vectorModelList
-  .map((item) => `| 索引-${item.name} | ${formatPrice(item.price, 1000)} |`)
-  .join('\n')}
-${chatModelList
-  .map((item) => `| 对话-${item.name} | ${formatPrice(item.price, 1000)} |`)
-  .join('\n')}
-| 文件QA拆分 | ${formatPrice(qaModel.price, 1000)} |`}
-            />
           </>
         )}
         {/* 付费二维码 */}

projects/app/src/pages/api/admin/initv43.ts

@@ -1,7 +1,7 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 

projects/app/src/pages/api/admin/initv44.ts

@@ -1,9 +1,10 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
-import { connectToDatabase, KB } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
-import { KbTypeEnum } from '@/constants/dataset';
+import { MongoDataset } from '@fastgpt/core/dataset/schema';
+import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 
@@ -12,13 +13,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     await connectToDatabase();
     await authUser({ req, authRoot: true });
 
-    await KB.updateMany(
+    await MongoDataset.updateMany(
       {
         type: { $exists: false }
       },
       {
         $set: {
-          type: KbTypeEnum.dataset,
+          type: DatasetTypeEnum.dataset,
           parentId: null
         }
       }

projects/app/src/pages/api/admin/initv441.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { connectToDatabase } from '@/service/mongo';
-import mongoose from 'mongoose';
+import mongoose from '@fastgpt/common/mongo';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 

projects/app/src/pages/api/admin/initv442.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { connectToDatabase, Bill } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/admin/initv445.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { connectToDatabase, App } from '@/service/mongo';
 import { FlowInputItemTypeEnum, FlowModuleTypeEnum } from '@/constants/flow';
 import { SystemInputEnum } from '@/constants/app';

projects/app/src/pages/api/admin/initv447.ts

@@ -1,11 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { connectToDatabase } from '@/service/mongo';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
-import mongoose, { Types } from 'mongoose';
+import { Types, connectionMongo } from '@fastgpt/common/mongo';
 import { delay } from '@/utils/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -71,7 +71,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 }
 
 async function init(rows: any[], initFileIds: string[]) {
-  const collection = mongoose.connection.db.collection(`dataset.files`);
+  const collection = connectionMongo.connection.db.collection(`dataset.files`);
 
   /* 遍历所有的 fileId，去找有没有对应的文件，没有的话则改成manual */
   const updateResult = await Promise.allSettled(

projects/app/src/pages/api/admin/sqlTest.ts

@@ -0,0 +1,45 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { authUser } from '@fastgpt/support/user/auth';
+import { connectToDatabase } from '@/service/mongo';
+import { PgClient } from '@/service/pg';
+import { PgDatasetTableName } from '@/constants/plugin';
+import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
+import { Types, connectionMongo } from '@fastgpt/common/mongo';
+import { delay } from '@/utils/tools';
+import { replaceVariable } from '@/utils/common/tools/text';
+import { getVector } from '../openapi/plugin/vector';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    // await connectToDatabase();
+    // const { text, analyze, sql } = req.body as {
+    //   userId: string;
+    //   text: string;
+    //   analyze?: boolean;
+    //   sql: string;
+    // };
+    // await authUser({ req, authRoot: true });
+
+    // const vectorModel = global.vectorModels[0];
+    // const { vectors } = await getVector({
+    //   model: vectorModel.model,
+    //   input: [text]
+    // });
+
+    // const start = Date.now();
+    // const result: any = await PgClient.query(sql.replace(/\[vector\]/g, `[${vectors[0]}]`));
+
+    jsonRes(res, {
+      data: {
+        // rows: result?.[2]?.rows,
+        // time: Date.now() - start
+      }
+    });
+  } catch (error) {
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}

projects/app/src/pages/api/app/create.ts

@@ -2,13 +2,14 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { App } from '@/service/models/app';
 import type { CreateAppParams } from '@/types/app';
 import { AppTypeEnum } from '@/constants/app';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const {
       name = 'APP',
       avatar,
@@ -23,8 +24,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     // 上限校验
     const authCount = await App.countDocuments({
       userId

projects/app/src/pages/api/app/data/totalUsage.ts

@@ -1,16 +1,15 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Bill } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
-import { Types } from 'mongoose';
+import { Types } from '@fastgpt/common/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    await connectToDatabase();
     const { appId, start, end } = req.body as { appId: string; start: number; end: number };
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     const result = await Bill.aggregate([
       {
         $match: {

projects/app/src/pages/api/app/del.ts

@@ -1,12 +1,14 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { Chat, App, connectToDatabase, Collection, OutLink } from '@/service/mongo';
+import { Chat, App, connectToDatabase, Collection } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { MongoOutLink } from '@fastgpt/support/outLink/schema';
+import { authUser } from '@fastgpt/support/user/auth';
 import { authApp } from '@/service/utils/auth';
 
 /* 获取我的模型 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const { appId } = req.query as { appId: string };
 
     if (!appId) {
@@ -16,8 +18,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     // 验证是否是该用户的 app
     await authApp({
       appId,
@@ -35,7 +35,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
 
     // 删除分享链接
-    await OutLink.deleteMany({
+    await MongoOutLink.deleteMany({
       appId
     });
 

projects/app/src/pages/api/app/detail.tsx

@@ -1,12 +1,13 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { authApp } from '@/service/utils/auth';
 
 /* 获取我的模型 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const { appId } = req.query as { appId: string };
 
     if (!appId) {
@@ -16,8 +17,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     const { app } = await authApp({
       appId,
       userId

projects/app/src/pages/api/app/getChatLogs.ts

@@ -1,15 +1,16 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Chat, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import type { PagingData } from '@/types';
 import { AppLogsListItemType } from '@/types/app';
-import { Types } from 'mongoose';
+import { Types } from '@fastgpt/common/mongo';
 import { addDays } from 'date-fns';
 import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    await connectToDatabase();
     const {
       pageNum = 1,
       pageSize = 20,
@@ -21,7 +22,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     if (!appId) {
       throw new Error('缺少参数');
     }
-    await connectToDatabase();
 
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });

projects/app/src/pages/api/app/myApps.ts

@@ -1,16 +1,15 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, App } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { AppListItemType } from '@/types/app';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     // 根据 userId 获取模型信息
     const myApps = await App.find(
       {

projects/app/src/pages/api/app/share/collection.ts

@@ -1,11 +1,12 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Collection, App } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 
 /* 模型收藏切换 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const { appId } = req.query as { appId: string };
 
     if (!appId) {
@@ -14,8 +15,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     const collectionRecord = await Collection.findOne({
       userId,
       modelId: appId

projects/app/src/pages/api/app/share/getModels.ts

@@ -3,20 +3,19 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase, App } from '@/service/mongo';
 import type { PagingData } from '@/types';
 import type { ShareAppItem } from '@/types/app';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
-import { Types } from 'mongoose';
+import { Types } from '@fastgpt/common/mongo';
 
 /* 获取模型列表 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const {
       searchText = '',
       pageNum = 1,
       pageSize = 20
     } = req.body as { searchText: string; pageNum: number; pageSize: number };
 
-    await connectToDatabase();
-
     const { userId } = await authUser({ req, authToken: true });
 
     const regex = new RegExp(searchText, 'i');

projects/app/src/pages/api/app/update.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { App } from '@/service/models/app';
 import type { AppUpdateParams } from '@/types/app';
 import { authApp } from '@/service/utils/auth';
@@ -9,6 +9,7 @@ import { authApp } from '@/service/utils/auth';
 /* 获取我的模型 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const { name, avatar, type, share, intro, modules } = req.body as AppUpdateParams;
     const { appId } = req.query as { appId: string };
 
@@ -19,8 +20,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     await authApp({
       appId,
       userId

projects/app/src/pages/api/chat/chatTest.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { sseErrRes } from '@/service/response';
 import { sseResponseEventEnum } from '@/constants/chat';
 import { sseResponse } from '@/service/utils/tools';
@@ -30,6 +30,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
   let { modules = [], history = [], prompt, variables = {}, appName, appId } = req.body as Props;
   try {
+    await connectToDatabase();
     if (!history || !modules || !prompt) {
       throw new Error('Prams Error');
     }
@@ -37,8 +38,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       throw new Error('history is not array');
     }
 
-    await connectToDatabase();
-
     /* user auth */
     const { userId, user } = await authUser({ req, authToken: true, authBalance: true });
 

projects/app/src/pages/api/chat/delChatRecordByContentId.ts

@@ -1,13 +1,12 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, ChatItem } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    const { chatId, contentId } = req.query as { chatId: string; contentId: string };
-
     await connectToDatabase();
+    const { chatId, contentId } = req.query as { chatId: string; contentId: string };
 
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });

projects/app/src/pages/api/chat/feedback/adminUpdate.ts

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, ChatItem } from '@/service/mongo';
 import type { AdminUpdateFeedbackParams } from '@/global/core/api/chatReq.d';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 
 /* 初始化我的聊天框，需要身份验证 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/chat/history/getHistory.ts

@@ -1,18 +1,17 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import type { ChatHistoryItemType } from '@/types/chat';
 import { ChatSourceEnum } from '@/constants/chat';
 
 /* 获取历史记录 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    await connectToDatabase();
     const { appId } = req.body as { appId?: string };
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     const data = await Chat.find(
       {
         userId,

projects/app/src/pages/api/chat/history/updateChatHistory.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 
 export type Props = {
   chatId: string;
@@ -12,12 +12,11 @@ export type Props = {
 /* 更新聊天标题 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    await connectToDatabase();
     const { chatId, customTitle, top } = req.body as Props;
 
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     await Chat.findOneAndUpdate(
       {
         chatId,

projects/app/src/pages/api/chat/init.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { Chat, ChatItem } from '@/service/mongo';
+import { Chat, ChatItem, connectToDatabase } from '@/service/mongo';
 import type { InitChatResponse } from '@/global/core/api/chatRes.d';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { ChatItemType } from '@/types/chat';
 import { authApp } from '@/service/utils/auth';
 import type { ChatSchema } from '@/types/mongoSchema';
@@ -12,6 +12,7 @@ import { TaskResponseKeyEnum } from '@/constants/chat';
 /* 初始化我的聊天框，需要身份验证 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    await connectToDatabase();
     const { userId } = await authUser({ req, authToken: true });
 
     let { appId, chatId } = req.query as {

projects/app/src/pages/api/chat/removeHistory.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat, ChatItem } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { ChatSourceEnum } from '@/constants/chat';
 
 type Props = {
@@ -12,11 +12,10 @@ type Props = {
 /* clear chat history */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    await connectToDatabase();
     const { chatId, appId } = req.query as Props;
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     if (chatId) {
       await Promise.all([
         Chat.findOneAndRemove({

projects/app/src/pages/api/common/bill/createTrainingBill.ts

@@ -1,18 +1,17 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Bill } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { BillSourceEnum } from '@/constants/user';
 import { CreateTrainingBillType } from '@/global/common/api/billReq.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    await connectToDatabase();
     const { name } = req.body as CreateTrainingBillType;
 
     const { userId } = await authUser({ req, authToken: true, authApiKey: true });
 
-    await connectToDatabase();
-
     const { _id } = await Bill.create({
       userId,
       appName: name,

projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import type { CreateQuestionGuideParams } from '@/global/core/api/aiReq.d';
 import { pushQuestionGuideBill } from '@/service/common/bill/push';
 import { defaultQGModel } from '@/pages/api/system/getInitData';

projects/app/src/pages/api/core/dataset/allDataset.ts

@@ -1,18 +1,18 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, KB } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { MongoDataset } from '@fastgpt/core/dataset/schema';
+import { authUser } from '@fastgpt/support/user/auth';
 import { getVectorModel } from '@/service/utils/data';
 import type { DatasetsItemType } from '@/types/core/dataset';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
+    const kbList = await MongoDataset.find({
-
-    const kbList = await KB.find({
       userId,
       type: 'dataset'
     });

projects/app/src/pages/api/core/dataset/create.ts

@@ -1,19 +1,19 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, KB } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { MongoDataset } from '@fastgpt/core/dataset/schema';
+import { authUser } from '@fastgpt/support/user/auth';
 import type { CreateDatasetParams } from '@/global/core/api/datasetReq.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateDatasetParams;
 
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
+    const { _id } = await MongoDataset.create({
-
-    const { _id } = await KB.create({
       name,
       userId,
       tags,

projects/app/src/pages/api/core/dataset/data/delDataById.ts

@@ -1,12 +1,14 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
 import { PgDatasetTableName } from '@/constants/plugin';
+import { connectToDatabase } from '@/service/mongo';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     let { dataId } = req.query as {
       dataId: string;
     };

projects/app/src/pages/api/core/dataset/data/exportAll.ts

@@ -1,7 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, User } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { MongoUser } from '@fastgpt/support/user/schema';
+import { authUser } from '@fastgpt/support/user/auth';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { findAllChildrenIds } from '../delete';
 import QueryStream from 'pg-query-stream';
@@ -11,6 +12,7 @@ import { responseWriteController } from '@/service/common/stream';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     let { kbId } = req.query as {
       kbId: string;
     };
@@ -19,8 +21,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       throw new Error('缺少参数');
     }
 
-    await connectToDatabase();
-
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
@@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     );
 
     // auth export times
-    const authTimes = await User.findOne(
+    const authTimes = await MongoUser.findOne(
       {
         _id: userId,
         $or: [
@@ -101,7 +101,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       stream.on('end', async () => {
         try {
           // update export time
-          await User.findByIdAndUpdate(userId, {
+          await MongoUser.findByIdAndUpdate(userId, {
             'limit.exportKbTime': new Date()
           });
         } catch (error) {}

projects/app/src/pages/api/core/dataset/data/getDataById.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import type { PgDataItemType } from '@/types/core/dataset/data';
@@ -15,6 +15,7 @@ export type Response = {
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     let { dataId } = req.query as {
       dataId: string;
     };
@@ -25,8 +26,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
-
     const where: any = [['user_id', userId], 'AND', ['id', dataId]];
 
     const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, {

projects/app/src/pages/api/core/dataset/data/getDataList.ts

@@ -1,13 +1,14 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import type { PgDataItemType } from '@/types/core/dataset/data';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     let {
       kbId,
       pageNum = 1,
@@ -28,7 +29,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
     searchText = searchText.replace(/'/g, '');
 
     const where: any = [

projects/app/src/pages/api/core/dataset/data/getQueueLen.ts

@@ -1,11 +1,12 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { TrainingData } from '@/service/mongo';
+import { TrainingData, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 
 /* 拆分数据成QA */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    await connectToDatabase();
     await authUser({ req, authToken: true });
 
     // split queue data

projects/app/src/pages/api/core/dataset/data/getTrainingData.ts

@@ -1,19 +1,19 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { TrainingModeEnum } from '@/constants/plugin';
-import { Types } from 'mongoose';
+import { Types } from '@fastgpt/common/mongo';
 import { startQueue } from '@/service/utils/tools';
 
 /* 拆分数据成QA */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
+    await connectToDatabase();
     const { kbId, init = false } = req.body as { kbId: string; init: boolean };
     if (!kbId) {
       throw new Error('参数错误');
     }
-    await connectToDatabase();
 
     const { userId } = await authUser({ req, authToken: true });
 

projects/app/src/pages/api/core/dataset/data/insertData.ts

@@ -5,7 +5,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authKb, authUser } from '@/service/utils/auth';
+import { authDataset } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { withNextCors } from '@/service/utils/tools';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { insertData2Dataset, PgClient } from '@/service/pg';
@@ -50,7 +51,7 @@ export async function getVectorAndInsertDataset(
   }
 
   // auth kb
-  const kb = await authKb({ kbId, userId });
+  const kb = await authDataset({ kbId, userId });
 
   const q = data?.q?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
   const a = data?.a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');

projects/app/src/pages/api/core/dataset/data/pushData.ts

@@ -1,9 +1,10 @@
 /* push data to training queue */
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, TrainingData, KB } from '@/service/mongo';
+import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authKb } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
+import { authDataset } from '@/service/utils/auth';
 import { withNextCors } from '@/service/utils/tools';
 import { TrainingModeEnum } from '@/constants/plugin';
 import { startQueue } from '@/service/utils/tools';
@@ -21,6 +22,7 @@ const modeMap = {
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const { kbId, data, mode = TrainingModeEnum.index } = req.body as PushDataProps;
 
     if (!kbId || !Array.isArray(data)) {
@@ -35,8 +37,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
       throw new Error('Data is too long, max 500');
     }
 
-    await connectToDatabase();
-
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true, authApiKey: true });
 
@@ -63,13 +63,13 @@ export async function pushDataToKb({
   billId
 }: { userId: string } & PushDataProps): Promise<PushDataResponse> {
   const [kb, vectorModel] = await Promise.all([
-    authKb({
+    authDataset({
       userId,
       kbId
     }),
     (async () => {
       if (mode === TrainingModeEnum.index) {
-        const vectorModel = (await KB.findById(kbId, 'vectorModel'))?.vectorModel;
+        const vectorModel = (await MongoDataset.findById(kbId, 'vectorModel'))?.vectorModel;
 
         return getVectorModel(vectorModel || global.vectorModels[0].model);
       }

projects/app/src/pages/api/core/dataset/data/updateData.ts

@@ -1,27 +1,27 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { withNextCors } from '@/service/utils/tools';
-import { KB, connectToDatabase } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDataset } from '@fastgpt/core/dataset/schema';
 import { getVector } from '@/pages/api/openapi/plugin/vector';
 import { PgDatasetTableName } from '@/constants/plugin';
 import type { UpdateDatasetDataPrams } from '@/global/core/api/datasetReq.d';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const { dataId, a = '', q = '', kbId } = req.body as UpdateDatasetDataPrams;
 
     if (!dataId) {
       throw new Error('缺少参数');
     }
 
-    await connectToDatabase();
-
     // auth user and get kb
     const [{ userId }, kb] = await Promise.all([
       authUser({ req, authToken: true }),
-      KB.findById(kbId, 'vectorModel')
+      MongoDataset.findById(kbId, 'vectorModel')
     ]);
 
     if (!kb) {

projects/app/src/pages/api/core/dataset/delete.ts

@@ -1,11 +1,12 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, KB, TrainingData } from '@/service/mongo';
+import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { MongoDataset } from '@fastgpt/core/dataset/schema';
+import { authUser } from '@fastgpt/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { GridFSStorage } from '@/service/lib/gridfs';
-import { Types } from 'mongoose';
+import { Types } from '@fastgpt/common/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -43,7 +44,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id)));
 
     // delete kb data
-    await KB.deleteMany({
+    await MongoDataset.deleteMany({
       _id: { $in: deletedIds },
       userId
     });
@@ -59,7 +60,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
 
 export async function findAllChildrenIds(id: string) {
   // find children
-  const children = await KB.find({ parentId: id });
+  const children = await MongoDataset.find({ parentId: id });
 
   let allChildrenIds = children.map((child) => String(child._id));
 

projects/app/src/pages/api/core/dataset/detail.ts

@@ -1,11 +1,13 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, KB } from '@/service/mongo';
+import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { getVectorModel } from '@/service/utils/data';
+import { MongoDataset } from '@fastgpt/core/dataset/schema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
+    await connectToDatabase();
     const { id } = req.query as {
       id: string;
     };
@@ -17,9 +19,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    await connectToDatabase();
+    const data = await MongoDataset.findOne({
-
-    const data = await KB.findOne({
       _id: id,
       userId
     });

projects/app/src/pages/api/core/dataset/file/delById.ts

@@ -1,11 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
-import { Types } from 'mongoose';
+import { Types } from '@fastgpt/common/mongo';
 import { isSpecialFileId } from '@fastgpt/core/dataset/utils';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/core/dataset/file/delEmptyFiles.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/core/dataset/file/detail.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
 import { datasetSpecialIdMap } from '@fastgpt/core/dataset/constant';
 import { datasetSpecialIds } from '@fastgpt/core/dataset/constant';

projects/app/src/pages/api/core/dataset/file/list.ts

@@ -1,18 +1,18 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { authUser } from '@/service/utils/auth';
+import { authUser } from '@fastgpt/support/user/auth';
 import { GridFSStorage } from '@/service/lib/gridfs';
 import { PgClient, updateDataFileId } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
-import { FileStatusEnum } from '@/constants/dataset';
 import { strIsLink } from '@fastgpt/common/tools/str';
 import {
+  FileStatusEnum,
   DatasetSpecialIdEnum,
   datasetSpecialIdMap,
   datasetSpecialIds
 } from '@fastgpt/core/dataset/constant';
-import { Types } from 'mongoose';
+import { Types } from '@fastgpt/common/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {