From b8a75921ed9a12ffa8ace28b1b2fead076e40e86 Mon Sep 17 00:00:00 2001 From: archer <545436317@qq.com> Date: Sat, 10 Jun 2023 22:56:57 +0800 Subject: [PATCH] feat: admin set env --- admin/.env.template | 2 + admin/service/route/system.js | 66 +++++++++++++++++++ admin/service/schema.js | 36 ++++++++-- admin/src/App.tsx | 12 +++- admin/src/fields.ts | 9 +++ .../src/pages/api/openapi/kb/appKbSearch.ts | 2 +- .../pages/api/openapi/text/sensitiveCheck.ts | 2 +- client/src/pages/api/system/updateEnv.ts | 32 +++++++++ client/src/service/events/generateQA.ts | 4 +- client/src/service/events/generateVector.ts | 4 +- client/src/service/models/system.ts | 30 +++++++++ client/src/service/mongo.ts | 40 ++++++----- client/src/service/pg.ts | 3 +- client/src/service/utils/auth.ts | 17 +++-- client/src/service/utils/tools.ts | 7 +- client/src/types/index.d.ts | 26 +++++--- docs/deploy/fastgpt/docker-compose.yml | 5 -- docs/dev/.env.template | 4 -- 18 files changed, 243 insertions(+), 58 deletions(-) create mode 100644 client/src/pages/api/system/updateEnv.ts create mode 100644 client/src/service/models/system.ts diff --git a/admin/.env.template b/admin/.env.template index 83423a8e3..d3dc1baee 100644 --- a/admin/.env.template +++ b/admin/.env.template @@ -3,4 +3,6 @@ MONGODB_NAME=fastgpt ADMIN_USER=username ADMIN_PASS=password ADMIN_SECRET=any +PARENT_URL=http://localhost:3000 # FastGpt服务的地址 +PARENT_ROOT_KEY=rootkey # FastGpt 的rootkey VITE_PUBLIC_SERVER_URL=http://localhost:3001 # 和server.js一致 \ No newline at end of file diff --git a/admin/service/route/system.js b/admin/service/route/system.js index 464939b7a..a3e3df6e5 100644 --- a/admin/service/route/system.js +++ b/admin/service/route/system.js @@ -1,4 +1,5 @@ import jwt from 'jsonwebtoken'; +import { System } from '../schema.js'; const adminAuth = { username: process.env.ADMIN_USER, @@ -6,6 +7,14 @@ const adminAuth = { }; const authSecret = process.env.ADMIN_SECRET; +const postParent = () => { + fetch(`${process.env.PARENT_URL}/api/system/updateEnv`, { + headers: { + rootkey: process.env.PARENT_ROOT_KEY + } + }); +}; + export const useSystemRoute = (app) => { app.post('/api/login', (req, res) => { if (!adminAuth.username || !adminAuth.password) { @@ -37,6 +46,63 @@ export const useSystemRoute = (app) => { res.status(401).end('username or password incorrect'); } }); + app.get('/system', auth(), async (req, res) => { + try { + const data = await System.find(); + const totalCount = await System.countDocuments(); + + res.header('Access-Control-Expose-Headers', 'X-Total-Count'); + res.header('X-Total-Count', totalCount); + res.json( + data.map((item) => { + const obj = item.toObject(); + return { + ...obj, + id: obj._id + }; + }) + ); + } catch (error) { + console.log(error); + + res.status(500).json({ error: 'Error creating system env' }); + } + }); + app.post('/system', auth(), async (req, res) => { + try { + await System.create({ + ...req.body, + sensitiveCheck: req.body.sensitiveCheck === 'true' + }); + postParent(); + res.json({}); + } catch (error) { + res.status(500).json({ error: 'Error creating system env' }); + } + }); + app.put('/system/:id', auth(), async (req, res) => { + try { + const _id = req.params.id; + await System.findByIdAndUpdate(_id, { + ...req.body, + sensitiveCheck: req.body.sensitiveCheck === 'true' + }); + postParent(); + res.json({}); + } catch (error) { + res.status(500).json({ error: 'Error updating system env' }); + } + }); + app.delete('/system/:id', auth(), async (req, res) => { + try { + const _id = req.params.id; + await System.findByIdAndDelete(_id); + + res.json({}); + } catch (error) { + res.status(500).json({ error: 'Error updating system env' }); + } + }); }; export const auth = () => { diff --git a/admin/service/schema.js b/admin/service/schema.js index 134313602..8de810cec 100644 --- a/admin/service/schema.js +++ b/admin/service/schema.js @@ -84,7 +84,35 @@ const modelSchema = new mongoose.Schema({ updateTime: Date }); -export const Model = mongoose.model('Model', modelSchema); -export const Kb = mongoose.model('Kb', kbSchema); -export const User = mongoose.model('User', userSchema, 'users'); -export const Pay = mongoose.model('Pay', paySchema, 'pays'); +const SystemSchema = new mongoose.Schema({ + openAIKeys: { + type: String, + default: '' + }, + openAITrainingKeys: { + type: String, + default: '' + }, + gpt4Key: { + type: String, + default: '' + }, + vectorMaxProcess: { + type: Number, + default: 10 + }, + qaMaxProcess: { + type: Number, + default: 10 + }, + sensitiveCheck: { + type: Boolean, + default: false + } +}); + +export const Model = mongoose.models['model'] || mongoose.model('model', modelSchema); +export const Kb = mongoose.models['kb'] || mongoose.model('kb', kbSchema); +export const User = mongoose.models['user'] || mongoose.model('user', userSchema); +export const Pay = mongoose.models['pay'] || mongoose.model('pay', paySchema); +export const System = mongoose.models['system'] || mongoose.model('system', SystemSchema); diff --git a/admin/src/App.tsx b/admin/src/App.tsx index 2781d9e0c..86e95b14c 100644 --- a/admin/src/App.tsx +++ b/admin/src/App.tsx @@ -7,7 +7,7 @@ import { fetchJSON } from 'tushan'; import { authProvider } from './auth'; -import { userFields, payFields, kbFields, ModelFields } from './fields'; +import { userFields, payFields, kbFields, ModelFields, SystemFields } from './fields'; import { Dashboard } from './Dashboard'; const authStorageKey = 'tushan:auth'; @@ -88,6 +88,16 @@ function App() { label="应用" list={} /> + + } + /> ); } diff --git a/admin/src/fields.ts b/admin/src/fields.ts index 42d4d437f..6657caf95 100644 --- a/admin/src/fields.ts +++ b/admin/src/fields.ts @@ -38,3 +38,12 @@ export const ModelFields = [ }), createTextField('temperature', { label: '温度' }) ]; + +export const SystemFields = [ + createTextField('openAIKeys', { label: 'openAIKeys，逗号隔开' }), + createTextField('openAITrainingKeys', { label: 'openAITrainingKeys' }), + createTextField('gpt4Key', { label: 'gpt4Key' }), + createTextField('vectorMaxProcess', { label: '向量最大进程' }), + createTextField('qaMaxProcess', { label: 'qa最大进程' }), + createTextField('sensitiveCheck', { label: '敏感词校验(true,false)' }) +]; diff --git a/client/src/pages/api/openapi/kb/appKbSearch.ts b/client/src/pages/api/openapi/kb/appKbSearch.ts index 3507e4cb8..f41252eeb 100644 --- a/client/src/pages/api/openapi/kb/appKbSearch.ts +++ b/client/src/pages/api/openapi/kb/appKbSearch.ts @@ -201,7 +201,7 @@ export async function appKbSearch({ searchPrompts: [ { obj: ChatRoleEnum.System, - value: `知识库:${systemPrompt}` + value: `知识库:<${systemPrompt}>` }, guidePrompt ] diff --git a/client/src/pages/api/openapi/text/sensitiveCheck.ts b/client/src/pages/api/openapi/text/sensitiveCheck.ts index 952900735..b7cec7e08 100644 --- a/client/src/pages/api/openapi/text/sensitiveCheck.ts +++ b/client/src/pages/api/openapi/text/sensitiveCheck.ts @@ -8,7 +8,7 @@ import { axiosConfig } from '@/service/utils/tools'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { - if (process.env.SENSITIVE_CHECK !== '1') { + if (global.systemEnv.sensitiveCheck) { return jsonRes(res); } diff --git a/client/src/pages/api/system/updateEnv.ts b/client/src/pages/api/system/updateEnv.ts new file mode 100644 index 000000000..1ed408db3 --- /dev/null +++ b/client/src/pages/api/system/updateEnv.ts @@ -0,0 +1,32 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@/service/response'; +import { System } from '@/service/models/system'; +import { authUser } from '@/service/utils/auth'; + +export type InitDateResponse = { + beianText: string; + googleVerKey: string; +}; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + await authUser({ req, authRoot: true }); + updateSystemEnv(); + jsonRes(res); +} + +export async function updateSystemEnv() { + try { + const mongoData = await System.findOne(); + + if (mongoData) { + const obj = mongoData.toObject(); + global.systemEnv = { + ...global.systemEnv, + ...obj + }; + } + console.log('update env', global.systemEnv); + } catch (error) { + console.log('update system env error'); + } +} diff --git a/client/src/service/events/generateQA.ts b/client/src/service/events/generateQA.ts index b095c354c..808623b16 100644 --- a/client/src/service/events/generateQA.ts +++ b/client/src/service/events/generateQA.ts @@ -16,9 +16,7 @@ const reduceQueue = () => { }; export async function generateQA(): Promise { - const maxProcess = Number(process.env.QA_MAX_PROCESS || 10); - - if (global.qaQueueLen >= maxProcess) return; + if (global.qaQueueLen >= global.systemEnv.qaMaxProcess) return; global.qaQueueLen++; let trainingId = ''; diff --git a/client/src/service/events/generateVector.ts b/client/src/service/events/generateVector.ts index 66e0d4865..937d3f20e 100644 --- a/client/src/service/events/generateVector.ts +++ b/client/src/service/events/generateVector.ts @@ -12,9 +12,7 @@ const reduceQueue = () => { /* 索引生成队列。每导入一次，就是一个单独的线程 */ export async function generateVector(): Promise { - const maxProcess = Number(process.env.VECTOR_MAX_PROCESS || 10); - - if (global.vectorQueueLen >= maxProcess) return; + if (global.vectorQueueLen >= global.systemEnv.vectorMaxProcess) return; global.vectorQueueLen++; let trainingId = ''; diff --git a/client/src/service/models/system.ts b/client/src/service/models/system.ts new file mode 100644 index 000000000..bbc62993e --- /dev/null +++ b/client/src/service/models/system.ts @@ -0,0 +1,30 @@ +import { Schema, model, models } from 'mongoose'; + +const SystemSchema = new Schema({ + openAIKeys: { + type: String, + default: '' + }, + openAITrainingKeys: { + type: String, + default: '' + }, + gpt4Key: { + type: String, + default: '' + }, + vectorMaxProcess: { + type: Number, + default: 10 + }, + qaMaxProcess: { + type: Number, + default: 10 + }, + sensitiveCheck: { + type: Boolean, + default: false + } +}); + +export const System = models['system'] || model('system', SystemSchema); diff --git a/client/src/service/mongo.ts b/client/src/service/mongo.ts index ae1c3b0a2..a25fba1ff 100644 --- a/client/src/service/mongo.ts +++ b/client/src/service/mongo.ts @@ -1,7 +1,7 @@ import mongoose from 'mongoose'; import tunnel from 'tunnel'; -import { TrainingData } from './mongo'; import { startQueue } from './utils/tools'; +import { updateSystemEnv } from '@/pages/api/system/updateEnv'; /** * 连接 MongoDB 数据库 @@ -11,6 +11,27 @@ export async function connectToDatabase(): Promise { return; } + // init global data + global.qaQueueLen = 0; + global.vectorQueueLen = 0; + global.systemEnv = { + openAIKeys: process.env.OPENAIKEY || '', + openAITrainingKeys: process.env.OPENAI_TRAINING_KEY || '', + gpt4Key: process.env.GPT4KEY || '', + vectorMaxProcess: 10, + qaMaxProcess: 10, + sensitiveCheck: false + }; + // proxy obj + if (process.env.AXIOS_PROXY_HOST && process.env.AXIOS_PROXY_PORT) { + global.httpsAgent = tunnel.httpsOverHttp({ + proxy: { + host: process.env.AXIOS_PROXY_HOST, + port: +process.env.AXIOS_PROXY_PORT + } + }); + } + global.mongodb = 'connecting'; try { mongoose.set('strictQuery', true); @@ -27,20 +48,8 @@ export async function connectToDatabase(): Promise { global.mongodb = null; } - // 创建代理对象 - if (process.env.AXIOS_PROXY_HOST && process.env.AXIOS_PROXY_PORT) { - global.httpsAgent = tunnel.httpsOverHttp({ - proxy: { - host: process.env.AXIOS_PROXY_HOST, - port: +process.env.AXIOS_PROXY_PORT - } - }); - } - - // 初始化队列 - global.qaQueueLen = 0; - global.vectorQueueLen = 0; - + // init function + updateSystemEnv(); startQueue(); } @@ -57,3 +66,4 @@ export * from './models/collection'; export * from './models/shareChat'; export * from './models/kb'; export * from './models/inform'; +export * from './models/system'; diff --git a/client/src/service/pg.ts b/client/src/service/pg.ts index 23bcedcac..8d817985c 100644 --- a/client/src/service/pg.ts +++ b/client/src/service/pg.ts @@ -6,14 +6,13 @@ export const connectPg = async () => { return global.pgClient; } - const maxLink = Number(process.env.VECTOR_MAX_PROCESS || 10); global.pgClient = new Pool({ host: process.env.PG_HOST, port: process.env.PG_PORT ? +process.env.PG_PORT : 5432, user: process.env.PG_USER, password: process.env.PG_PASSWORD, database: process.env.PG_DB_NAME, - max: Math.floor(maxLink * 0.5), + max: 80, idleTimeoutMillis: 60000, connectionTimeoutMillis: 20000 }); diff --git a/client/src/service/utils/auth.ts b/client/src/service/utils/auth.ts index 168cde228..4ffde9045 100644 --- a/client/src/service/utils/auth.ts +++ b/client/src/service/utils/auth.ts @@ -123,14 +123,21 @@ export const authUser = async ({ export const getSystemOpenAiKey = (type: ApiKeyType) => { const keys = (() => { if (type === 'training') { - return process.env.OPENAI_TRAINING_KEY?.split(',') || []; + return global.systemEnv.openAITrainingKeys?.split(',') || []; } - return process.env.OPENAIKEY?.split(',') || []; + return global.systemEnv.openAIKeys?.split(',') || []; })(); // 纯字符串类型 const i = Math.floor(Math.random() * keys.length); - return keys[i] || (process.env.OPENAIKEY as string); + return keys[i] || (global.systemEnv.openAIKeys as string); +}; +export const getGpt4Key = () => { + const keys = global.systemEnv.gpt4Key?.split(',') || []; + + // 纯字符串类型 + const i = Math.floor(Math.random() * keys.length); + return keys[i] || (global.systemEnv.openAIKeys as string); }; /* 获取 api 请求的 key */ @@ -157,11 +164,11 @@ export const getApiKey = async ({ }, [OpenAiChatEnum.GPT4]: { userOpenAiKey: user.openaiKey || '', - systemAuthKey: process.env.GPT4KEY as string + systemAuthKey: getGpt4Key() as string }, [OpenAiChatEnum.GPT432k]: { userOpenAiKey: user.openaiKey || '', - systemAuthKey: process.env.GPT4KEY as string + systemAuthKey: getGpt4Key() as string }, [ClaudeEnum.Claude]: { userOpenAiKey: '', diff --git a/client/src/service/utils/tools.ts b/client/src/service/utils/tools.ts index b33f2047f..95dc437ad 100644 --- a/client/src/service/utils/tools.ts +++ b/client/src/service/utils/tools.ts @@ -60,13 +60,10 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler { } export const startQueue = () => { - const qaMax = Number(process.env.QA_MAX_PROCESS || 10); - const vectorMax = Number(process.env.VECTOR_MAX_PROCESS || 10); - - for (let i = 0; i < qaMax; i++) { + for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) { generateQA(); } - for (let i = 0; i < vectorMax; i++) { + for (let i = 0; i < global.systemEnv.vectorMaxProcess; i++) { generateVector(); } }; diff --git a/client/src/types/index.d.ts b/client/src/types/index.d.ts index 4612d2074..11d15fea3 100644 --- a/client/src/types/index.d.ts +++ b/client/src/types/index.d.ts @@ -3,6 +3,15 @@ import type { Agent } from 'http'; import type { Pool } from 'pg'; import type { Tiktoken } from '@dqbd/tiktoken'; +export type PagingData = { + pageNum: number; + pageSize: number; + data: T[]; + total?: number; +}; + +export type RequestPaging = { pageNum: number; pageSize: number; [key]: any }; + declare global { var mongodb: Mongoose | string | null; var pgClient: Pool | null; @@ -13,17 +22,16 @@ declare global { var qaQueueLen: number; var vectorQueueLen: number; var OpenAiEncMap: Record; + var systemEnv: { + openAIKeys: string; + openAITrainingKeys: string; + gpt4Key: string; + vectorMaxProcess: number; + qaMaxProcess: number; + sensitiveCheck: boolean; + }; interface Window { ['pdfjs-dist/build/pdf']: any; } } - -export type PagingData = { - pageNum: number; - pageSize: number; - data: T[]; - total?: number; -}; - -export type RequestPaging = { pageNum: number; pageSize: number; [key]: any }; diff --git a/docs/deploy/fastgpt/docker-compose.yml b/docs/deploy/fastgpt/docker-compose.yml index da5e9f3ea..672b6b5a5 100644 --- a/docs/deploy/fastgpt/docker-compose.yml +++ b/docs/deploy/fastgpt/docker-compose.yml @@ -55,15 +55,10 @@ services: # google V3 安全校验（可选） - CLIENT_GOOGLE_VER_TOKEN=xxx - SERVICE_GOOGLE_VER_TOKEN=xx - # QA和向量生成最大进程数 - - QA_MAX_PROCESS=10 - - VECTOR_MAX_PROCESS=10 # token加密凭证（随便填，作为登录凭证） - TOKEN_KEY=xxxx # root key, 最高权限，可以内部接口互相调用 - ROOT_KEY=xxx - # 是否进行内容安全校验(1: 开启，0: 关闭) - - SENSITIVE_CHECK=1 # 和上方mongo镜像的username,password对应 - MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin - MONGODB_NAME=fastgpt diff --git a/docs/dev/.env.template b/docs/dev/.env.template index 361ac452d..54fc1f659 100644 --- a/docs/dev/.env.template +++ b/docs/dev/.env.template @@ -1,5 +1,3 @@ -QA_MAX_PROCESS=30 -VECTOR_MAX_PROCESS=30 # 运行端口，如果不是 3000 口运行，需要改成其他的。注意：不是改了这个变量就会变成其他端口，而是因为改成其他端口，才用这个变量。 PORT=3000 # 代理 @@ -17,8 +15,6 @@ aliTemplateCode=xxxx TOKEN_KEY=dfdasfdas # root key, 最高权限 ROOT_KEY=fdafasd -# 是否进行安全校验(1: 开启，0: 关闭) -SENSITIVE_CHECK=0 # openai # OPENAI_BASE_URL=http://ai.openai.com/v1 # OPENAI_BASE_URL_AUTH=可选安全凭证，会放到 header.auth 里