diff --git a/admin/.env.template b/admin/.env.template
index 83423a8e3..d3dc1baee 100644
--- a/admin/.env.template
+++ b/admin/.env.template
@@ -3,4 +3,6 @@ MONGODB_NAME=fastgpt
ADMIN_USER=username
ADMIN_PASS=password
ADMIN_SECRET=any
+PARENT_URL=http://localhost:3000 # FastGpt服务的地址
+PARENT_ROOT_KEY=rootkey # FastGpt 的rootkey
VITE_PUBLIC_SERVER_URL=http://localhost:3001 # 和server.js一致
\ No newline at end of file
diff --git a/admin/service/route/system.js b/admin/service/route/system.js
index 464939b7a..a3e3df6e5 100644
--- a/admin/service/route/system.js
+++ b/admin/service/route/system.js
@@ -1,4 +1,5 @@
import jwt from 'jsonwebtoken';
+import { System } from '../schema.js';
const adminAuth = {
username: process.env.ADMIN_USER,
@@ -6,6 +7,14 @@ const adminAuth = {
};
const authSecret = process.env.ADMIN_SECRET;
+const postParent = () => {
+ fetch(`${process.env.PARENT_URL}/api/system/updateEnv`, {
+ headers: {
+ rootkey: process.env.PARENT_ROOT_KEY
+ }
+ });
+};
+
export const useSystemRoute = (app) => {
app.post('/api/login', (req, res) => {
if (!adminAuth.username || !adminAuth.password) {
@@ -37,6 +46,63 @@ export const useSystemRoute = (app) => {
res.status(401).end('username or password incorrect');
}
});
+ app.get('/system', auth(), async (req, res) => {
+ try {
+ const data = await System.find();
+ const totalCount = await System.countDocuments();
+
+ res.header('Access-Control-Expose-Headers', 'X-Total-Count');
+ res.header('X-Total-Count', totalCount);
+ res.json(
+ data.map((item) => {
+ const obj = item.toObject();
+ return {
+ ...obj,
+ id: obj._id
+ };
+ })
+ );
+ } catch (error) {
+ console.log(error);
+
+ res.status(500).json({ error: 'Error creating system env' });
+ }
+ });
+ app.post('/system', auth(), async (req, res) => {
+ try {
+ await System.create({
+ ...req.body,
+ sensitiveCheck: req.body.sensitiveCheck === 'true'
+ });
+ postParent();
+ res.json({});
+ } catch (error) {
+ res.status(500).json({ error: 'Error creating system env' });
+ }
+ });
+ app.put('/system/:id', auth(), async (req, res) => {
+ try {
+ const _id = req.params.id;
+ await System.findByIdAndUpdate(_id, {
+ ...req.body,
+ sensitiveCheck: req.body.sensitiveCheck === 'true'
+ });
+ postParent();
+ res.json({});
+ } catch (error) {
+ res.status(500).json({ error: 'Error updating system env' });
+ }
+ });
+ app.delete('/system/:id', auth(), async (req, res) => {
+ try {
+ const _id = req.params.id;
+ await System.findByIdAndDelete(_id);
+
+ res.json({});
+ } catch (error) {
+ res.status(500).json({ error: 'Error updating system env' });
+ }
+ });
};
export const auth = () => {
diff --git a/admin/service/schema.js b/admin/service/schema.js
index 134313602..8de810cec 100644
--- a/admin/service/schema.js
+++ b/admin/service/schema.js
@@ -84,7 +84,35 @@ const modelSchema = new mongoose.Schema({
updateTime: Date
});
-export const Model = mongoose.model('Model', modelSchema);
-export const Kb = mongoose.model('Kb', kbSchema);
-export const User = mongoose.model('User', userSchema, 'users');
-export const Pay = mongoose.model('Pay', paySchema, 'pays');
+const SystemSchema = new mongoose.Schema({
+ openAIKeys: {
+ type: String,
+ default: ''
+ },
+ openAITrainingKeys: {
+ type: String,
+ default: ''
+ },
+ gpt4Key: {
+ type: String,
+ default: ''
+ },
+ vectorMaxProcess: {
+ type: Number,
+ default: 10
+ },
+ qaMaxProcess: {
+ type: Number,
+ default: 10
+ },
+ sensitiveCheck: {
+ type: Boolean,
+ default: false
+ }
+});
+
+export const Model = mongoose.models['model'] || mongoose.model('model', modelSchema);
+export const Kb = mongoose.models['kb'] || mongoose.model('kb', kbSchema);
+export const User = mongoose.models['user'] || mongoose.model('user', userSchema);
+export const Pay = mongoose.models['pay'] || mongoose.model('pay', paySchema);
+export const System = mongoose.models['system'] || mongoose.model('system', SystemSchema);
diff --git a/admin/src/App.tsx b/admin/src/App.tsx
index 2781d9e0c..86e95b14c 100644
--- a/admin/src/App.tsx
+++ b/admin/src/App.tsx
@@ -7,7 +7,7 @@ import {
fetchJSON
} from 'tushan';
import { authProvider } from './auth';
-import { userFields, payFields, kbFields, ModelFields } from './fields';
+import { userFields, payFields, kbFields, ModelFields, SystemFields } from './fields';
import { Dashboard } from './Dashboard';
const authStorageKey = 'tushan:auth';
@@ -88,6 +88,16 @@ function App() {
label="应用"
list={}
/>
+
+ }
+ />
);
}
diff --git a/admin/src/fields.ts b/admin/src/fields.ts
index 42d4d437f..6657caf95 100644
--- a/admin/src/fields.ts
+++ b/admin/src/fields.ts
@@ -38,3 +38,12 @@ export const ModelFields = [
}),
createTextField('temperature', { label: '温度' })
];
+
+export const SystemFields = [
+ createTextField('openAIKeys', { label: 'openAIKeys,逗号隔开' }),
+ createTextField('openAITrainingKeys', { label: 'openAITrainingKeys' }),
+ createTextField('gpt4Key', { label: 'gpt4Key' }),
+ createTextField('vectorMaxProcess', { label: '向量最大进程' }),
+ createTextField('qaMaxProcess', { label: 'qa最大进程' }),
+ createTextField('sensitiveCheck', { label: '敏感词校验(true,false)' })
+];
diff --git a/client/src/pages/api/openapi/kb/appKbSearch.ts b/client/src/pages/api/openapi/kb/appKbSearch.ts
index 3507e4cb8..f41252eeb 100644
--- a/client/src/pages/api/openapi/kb/appKbSearch.ts
+++ b/client/src/pages/api/openapi/kb/appKbSearch.ts
@@ -201,7 +201,7 @@ export async function appKbSearch({
searchPrompts: [
{
obj: ChatRoleEnum.System,
- value: `知识库:${systemPrompt}`
+ value: `知识库:<${systemPrompt}>`
},
guidePrompt
]
diff --git a/client/src/pages/api/openapi/text/sensitiveCheck.ts b/client/src/pages/api/openapi/text/sensitiveCheck.ts
index 952900735..b7cec7e08 100644
--- a/client/src/pages/api/openapi/text/sensitiveCheck.ts
+++ b/client/src/pages/api/openapi/text/sensitiveCheck.ts
@@ -8,7 +8,7 @@ import { axiosConfig } from '@/service/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
- if (process.env.SENSITIVE_CHECK !== '1') {
+ if (global.systemEnv.sensitiveCheck) {
return jsonRes(res);
}
diff --git a/client/src/pages/api/system/updateEnv.ts b/client/src/pages/api/system/updateEnv.ts
new file mode 100644
index 000000000..1ed408db3
--- /dev/null
+++ b/client/src/pages/api/system/updateEnv.ts
@@ -0,0 +1,32 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { System } from '@/service/models/system';
+import { authUser } from '@/service/utils/auth';
+
+export type InitDateResponse = {
+ beianText: string;
+ googleVerKey: string;
+};
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+ await authUser({ req, authRoot: true });
+ updateSystemEnv();
+ jsonRes(res);
+}
+
+export async function updateSystemEnv() {
+ try {
+ const mongoData = await System.findOne();
+
+ if (mongoData) {
+ const obj = mongoData.toObject();
+ global.systemEnv = {
+ ...global.systemEnv,
+ ...obj
+ };
+ }
+ console.log('update env', global.systemEnv);
+ } catch (error) {
+ console.log('update system env error');
+ }
+}
diff --git a/client/src/service/events/generateQA.ts b/client/src/service/events/generateQA.ts
index b095c354c..808623b16 100644
--- a/client/src/service/events/generateQA.ts
+++ b/client/src/service/events/generateQA.ts
@@ -16,9 +16,7 @@ const reduceQueue = () => {
};
export async function generateQA(): Promise {
- const maxProcess = Number(process.env.QA_MAX_PROCESS || 10);
-
- if (global.qaQueueLen >= maxProcess) return;
+ if (global.qaQueueLen >= global.systemEnv.qaMaxProcess) return;
global.qaQueueLen++;
let trainingId = '';
diff --git a/client/src/service/events/generateVector.ts b/client/src/service/events/generateVector.ts
index 66e0d4865..937d3f20e 100644
--- a/client/src/service/events/generateVector.ts
+++ b/client/src/service/events/generateVector.ts
@@ -12,9 +12,7 @@ const reduceQueue = () => {
/* 索引生成队列。每导入一次,就是一个单独的线程 */
export async function generateVector(): Promise {
- const maxProcess = Number(process.env.VECTOR_MAX_PROCESS || 10);
-
- if (global.vectorQueueLen >= maxProcess) return;
+ if (global.vectorQueueLen >= global.systemEnv.vectorMaxProcess) return;
global.vectorQueueLen++;
let trainingId = '';
diff --git a/client/src/service/models/system.ts b/client/src/service/models/system.ts
new file mode 100644
index 000000000..bbc62993e
--- /dev/null
+++ b/client/src/service/models/system.ts
@@ -0,0 +1,30 @@
+import { Schema, model, models } from 'mongoose';
+
+const SystemSchema = new Schema({
+ openAIKeys: {
+ type: String,
+ default: ''
+ },
+ openAITrainingKeys: {
+ type: String,
+ default: ''
+ },
+ gpt4Key: {
+ type: String,
+ default: ''
+ },
+ vectorMaxProcess: {
+ type: Number,
+ default: 10
+ },
+ qaMaxProcess: {
+ type: Number,
+ default: 10
+ },
+ sensitiveCheck: {
+ type: Boolean,
+ default: false
+ }
+});
+
+export const System = models['system'] || model('system', SystemSchema);
diff --git a/client/src/service/mongo.ts b/client/src/service/mongo.ts
index ae1c3b0a2..a25fba1ff 100644
--- a/client/src/service/mongo.ts
+++ b/client/src/service/mongo.ts
@@ -1,7 +1,7 @@
import mongoose from 'mongoose';
import tunnel from 'tunnel';
-import { TrainingData } from './mongo';
import { startQueue } from './utils/tools';
+import { updateSystemEnv } from '@/pages/api/system/updateEnv';
/**
* 连接 MongoDB 数据库
@@ -11,6 +11,27 @@ export async function connectToDatabase(): Promise {
return;
}
+ // init global data
+ global.qaQueueLen = 0;
+ global.vectorQueueLen = 0;
+ global.systemEnv = {
+ openAIKeys: process.env.OPENAIKEY || '',
+ openAITrainingKeys: process.env.OPENAI_TRAINING_KEY || '',
+ gpt4Key: process.env.GPT4KEY || '',
+ vectorMaxProcess: 10,
+ qaMaxProcess: 10,
+ sensitiveCheck: false
+ };
+ // proxy obj
+ if (process.env.AXIOS_PROXY_HOST && process.env.AXIOS_PROXY_PORT) {
+ global.httpsAgent = tunnel.httpsOverHttp({
+ proxy: {
+ host: process.env.AXIOS_PROXY_HOST,
+ port: +process.env.AXIOS_PROXY_PORT
+ }
+ });
+ }
+
global.mongodb = 'connecting';
try {
mongoose.set('strictQuery', true);
@@ -27,20 +48,8 @@ export async function connectToDatabase(): Promise {
global.mongodb = null;
}
- // 创建代理对象
- if (process.env.AXIOS_PROXY_HOST && process.env.AXIOS_PROXY_PORT) {
- global.httpsAgent = tunnel.httpsOverHttp({
- proxy: {
- host: process.env.AXIOS_PROXY_HOST,
- port: +process.env.AXIOS_PROXY_PORT
- }
- });
- }
-
- // 初始化队列
- global.qaQueueLen = 0;
- global.vectorQueueLen = 0;
-
+ // init function
+ updateSystemEnv();
startQueue();
}
@@ -57,3 +66,4 @@ export * from './models/collection';
export * from './models/shareChat';
export * from './models/kb';
export * from './models/inform';
+export * from './models/system';
diff --git a/client/src/service/pg.ts b/client/src/service/pg.ts
index 23bcedcac..8d817985c 100644
--- a/client/src/service/pg.ts
+++ b/client/src/service/pg.ts
@@ -6,14 +6,13 @@ export const connectPg = async () => {
return global.pgClient;
}
- const maxLink = Number(process.env.VECTOR_MAX_PROCESS || 10);
global.pgClient = new Pool({
host: process.env.PG_HOST,
port: process.env.PG_PORT ? +process.env.PG_PORT : 5432,
user: process.env.PG_USER,
password: process.env.PG_PASSWORD,
database: process.env.PG_DB_NAME,
- max: Math.floor(maxLink * 0.5),
+ max: 80,
idleTimeoutMillis: 60000,
connectionTimeoutMillis: 20000
});
diff --git a/client/src/service/utils/auth.ts b/client/src/service/utils/auth.ts
index 168cde228..4ffde9045 100644
--- a/client/src/service/utils/auth.ts
+++ b/client/src/service/utils/auth.ts
@@ -123,14 +123,21 @@ export const authUser = async ({
export const getSystemOpenAiKey = (type: ApiKeyType) => {
const keys = (() => {
if (type === 'training') {
- return process.env.OPENAI_TRAINING_KEY?.split(',') || [];
+ return global.systemEnv.openAITrainingKeys?.split(',') || [];
}
- return process.env.OPENAIKEY?.split(',') || [];
+ return global.systemEnv.openAIKeys?.split(',') || [];
})();
// 纯字符串类型
const i = Math.floor(Math.random() * keys.length);
- return keys[i] || (process.env.OPENAIKEY as string);
+ return keys[i] || (global.systemEnv.openAIKeys as string);
+};
+export const getGpt4Key = () => {
+ const keys = global.systemEnv.gpt4Key?.split(',') || [];
+
+ // 纯字符串类型
+ const i = Math.floor(Math.random() * keys.length);
+ return keys[i] || (global.systemEnv.openAIKeys as string);
};
/* 获取 api 请求的 key */
@@ -157,11 +164,11 @@ export const getApiKey = async ({
},
[OpenAiChatEnum.GPT4]: {
userOpenAiKey: user.openaiKey || '',
- systemAuthKey: process.env.GPT4KEY as string
+ systemAuthKey: getGpt4Key() as string
},
[OpenAiChatEnum.GPT432k]: {
userOpenAiKey: user.openaiKey || '',
- systemAuthKey: process.env.GPT4KEY as string
+ systemAuthKey: getGpt4Key() as string
},
[ClaudeEnum.Claude]: {
userOpenAiKey: '',
diff --git a/client/src/service/utils/tools.ts b/client/src/service/utils/tools.ts
index b33f2047f..95dc437ad 100644
--- a/client/src/service/utils/tools.ts
+++ b/client/src/service/utils/tools.ts
@@ -60,13 +60,10 @@ export function withNextCors(handler: NextApiHandler): NextApiHandler {
}
export const startQueue = () => {
- const qaMax = Number(process.env.QA_MAX_PROCESS || 10);
- const vectorMax = Number(process.env.VECTOR_MAX_PROCESS || 10);
-
- for (let i = 0; i < qaMax; i++) {
+ for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) {
generateQA();
}
- for (let i = 0; i < vectorMax; i++) {
+ for (let i = 0; i < global.systemEnv.vectorMaxProcess; i++) {
generateVector();
}
};
diff --git a/client/src/types/index.d.ts b/client/src/types/index.d.ts
index 4612d2074..11d15fea3 100644
--- a/client/src/types/index.d.ts
+++ b/client/src/types/index.d.ts
@@ -3,6 +3,15 @@ import type { Agent } from 'http';
import type { Pool } from 'pg';
import type { Tiktoken } from '@dqbd/tiktoken';
+export type PagingData = {
+ pageNum: number;
+ pageSize: number;
+ data: T[];
+ total?: number;
+};
+
+export type RequestPaging = { pageNum: number; pageSize: number; [key]: any };
+
declare global {
var mongodb: Mongoose | string | null;
var pgClient: Pool | null;
@@ -13,17 +22,16 @@ declare global {
var qaQueueLen: number;
var vectorQueueLen: number;
var OpenAiEncMap: Record;
+ var systemEnv: {
+ openAIKeys: string;
+ openAITrainingKeys: string;
+ gpt4Key: string;
+ vectorMaxProcess: number;
+ qaMaxProcess: number;
+ sensitiveCheck: boolean;
+ };
interface Window {
['pdfjs-dist/build/pdf']: any;
}
}
-
-export type PagingData = {
- pageNum: number;
- pageSize: number;
- data: T[];
- total?: number;
-};
-
-export type RequestPaging = { pageNum: number; pageSize: number; [key]: any };
diff --git a/docs/deploy/fastgpt/docker-compose.yml b/docs/deploy/fastgpt/docker-compose.yml
index da5e9f3ea..672b6b5a5 100644
--- a/docs/deploy/fastgpt/docker-compose.yml
+++ b/docs/deploy/fastgpt/docker-compose.yml
@@ -55,15 +55,10 @@ services:
# google V3 安全校验(可选)
- CLIENT_GOOGLE_VER_TOKEN=xxx
- SERVICE_GOOGLE_VER_TOKEN=xx
- # QA和向量生成最大进程数
- - QA_MAX_PROCESS=10
- - VECTOR_MAX_PROCESS=10
# token加密凭证(随便填,作为登录凭证)
- TOKEN_KEY=xxxx
# root key, 最高权限,可以内部接口互相调用
- ROOT_KEY=xxx
- # 是否进行内容安全校验(1: 开启,0: 关闭)
- - SENSITIVE_CHECK=1
# 和上方mongo镜像的username,password对应
- MONGODB_URI=mongodb://username:password@0.0.0.0:27017/?authSource=admin
- MONGODB_NAME=fastgpt
diff --git a/docs/dev/.env.template b/docs/dev/.env.template
index 361ac452d..54fc1f659 100644
--- a/docs/dev/.env.template
+++ b/docs/dev/.env.template
@@ -1,5 +1,3 @@
-QA_MAX_PROCESS=30
-VECTOR_MAX_PROCESS=30
# 运行端口,如果不是 3000 口运行,需要改成其他的。注意:不是改了这个变量就会变成其他端口,而是因为改成其他端口,才用这个变量。
PORT=3000
# 代理
@@ -17,8 +15,6 @@ aliTemplateCode=xxxx
TOKEN_KEY=dfdasfdas
# root key, 最高权限
ROOT_KEY=fdafasd
-# 是否进行安全校验(1: 开启,0: 关闭)
-SENSITIVE_CHECK=0
# openai
# OPENAI_BASE_URL=http://ai.openai.com/v1
# OPENAI_BASE_URL_AUTH=可选安全凭证,会放到 header.auth 里