This commit is contained in:
Archer
2023-11-09 09:46:57 +08:00
committed by GitHub
parent 661ee79943
commit 8bb5588305
402 changed files with 9899 additions and 5967 deletions

View File

@@ -18,7 +18,6 @@ function requestStart(config: InternalAxiosRequestConfig): InternalAxiosRequestC
if (config.headers) {
config.headers.rootkey = process.env.ROOT_KEY;
}
return config;
}
@@ -62,7 +61,8 @@ function responseError(err: any) {
const instance = axios.create({
timeout: 60000, // 超时时间
headers: {
'content-type': 'application/json'
'content-type': 'application/json',
'Cache-Control': 'no-cache'
}
});
@@ -73,7 +73,7 @@ instance.interceptors.response.use(responseSuccess, (err) => Promise.reject(err)
export function request(url: string, data: any, config: ConfigType, method: Method): any {
if (!global.systemEnv?.pluginBaseUrl) {
return Promise.reject('商业版插件加载中...');
return Promise.reject('该功能为商业版特有...');
}
/* 去空 */

View File

@@ -0,0 +1,111 @@
import { Types, connectionMongo } from '../../mongo';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import fsp from 'fs/promises';
import fs from 'fs';
import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
export function getGFSCollection(bucket: `${BucketNameEnum}`) {
return connectionMongo.connection.db.collection(`${bucket}.files`);
}
export function getGridBucket(bucket: `${BucketNameEnum}`) {
return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db, {
bucketName: bucket
});
}
/* crud file */
export async function uploadFile({
bucketName,
teamId,
tmbId,
path,
filename,
metadata = {}
}: {
bucketName: `${BucketNameEnum}`;
teamId: string;
tmbId: string;
path: string;
filename: string;
metadata?: Record<string, any>;
}) {
if (!path) return Promise.reject(`filePath is empty`);
if (!filename) return Promise.reject(`filename is empty`);
const stats = await fsp.stat(path);
if (!stats.isFile()) return Promise.reject(`${path} is not a file`);
metadata.teamId = teamId;
metadata.tmbId = tmbId;
// create a gridfs bucket
const bucket = getGridBucket(bucketName);
const stream = bucket.openUploadStream(filename, {
metadata,
contentType: metadata?.contentType
});
// save to gridfs
await new Promise((resolve, reject) => {
fs.createReadStream(path)
.pipe(stream as any)
.on('finish', resolve)
.on('error', reject);
});
return String(stream.id);
}
export async function getFileById({
bucketName,
fileId
}: {
bucketName: `${BucketNameEnum}`;
fileId: string;
}) {
const db = getGFSCollection(bucketName);
const file = await db.findOne<DatasetFileSchema>({
_id: new Types.ObjectId(fileId)
});
if (!file) {
return Promise.reject('File not found');
}
return file;
}
export async function delFileById({
bucketName,
fileId
}: {
bucketName: `${BucketNameEnum}`;
fileId: string;
}) {
const bucket = getGridBucket(bucketName);
await bucket.delete(new Types.ObjectId(fileId));
return true;
}
export async function getDownloadBuf({
bucketName,
fileId
}: {
bucketName: `${BucketNameEnum}`;
fileId: string;
}) {
const bucket = getGridBucket(bucketName);
const stream = bucket.openDownloadStream(new Types.ObjectId(fileId));
const buf: Buffer = await new Promise((resolve, reject) => {
const buffers: Buffer[] = [];
stream.on('data', (data) => buffers.push(data));
stream.on('error', reject);
stream.on('end', () => resolve(Buffer.concat(buffers)));
});
return buf;
}

View File

@@ -0,0 +1,26 @@
/* add logger */
export const addLog = {
info: (msg: string, obj?: Record<string, any>) => {
global.logger?.info(msg, { meta: obj });
},
error: (msg: string, error?: any) => {
global.logger?.error(msg, {
meta: {
stack: error?.stack,
...(error?.config && {
config: {
headers: error.config.headers,
url: error.config.url,
data: error.config.data
}
}),
...(error?.response && {
response: {
status: error.response.status,
statusText: error.response.statusText
}
})
}
});
}
};

View File

@@ -1,6 +1,6 @@
import mongoose from './index';
import 'winston-mongodb';
import { createLogger, format, transports } from 'winston';
import 'winston-mongodb';
/**
* connect MongoDB and init data
@@ -19,9 +19,6 @@ export async function connectMongo({
beforeHook && (await beforeHook());
// logger
initLogger();
console.log('mongo start connect');
try {
mongoose.set('strictQuery', true);
@@ -35,9 +32,11 @@ export async function connectMongo({
});
console.log('mongo connected');
initLogger();
afterHook && (await afterHook());
} catch (error) {
global.mongodb.disconnect();
console.log('error->', 'mongo connect error', error);
global.mongodb = undefined;
}

View File

@@ -1,39 +1,21 @@
import mongoose from './index';
import mongoose, { connectionMongo } from './index';
export class MongoSession {
tasks: (() => Promise<any>)[] = [];
session: mongoose.mongo.ClientSession | null = null;
opts: {
session: mongoose.mongo.ClientSession;
new: boolean;
} | null = null;
export async function mongoSessionTask(
fn: (session: mongoose.mongo.ClientSession) => Promise<any>
) {
const session = await connectionMongo.startSession();
constructor() {}
async init() {
this.session = await mongoose.startSession();
this.opts = { session: this.session, new: true };
}
push(
tasks: ((opts: {
session: mongoose.mongo.ClientSession;
new: boolean;
}) => () => Promise<any>)[] = []
) {
if (!this.opts) return;
// this.tasks = this.tasks.concat(tasks.map((item) => item(this.opts)));
}
async run() {
if (!this.session || !this.opts) return;
try {
this.session.startTransaction();
try {
session.startTransaction();
const opts = { session: this.session, new: true };
await fn(session);
await this.session.commitTransaction();
} catch (error) {
await this.session.abortTransaction();
console.error(error);
}
this.session.endSession();
await session.commitTransaction();
await session.endSession();
} catch (error) {
await session.abortTransaction();
await session.endSession();
console.error(error);
return Promise.reject(error);
}
}

View File

@@ -0,0 +1,187 @@
import { Pool } from 'pg';
import type { QueryResultRow } from 'pg';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
export const connectPg = async (): Promise<Pool> => {
if (global.pgClient) {
return global.pgClient;
}
global.pgClient = new Pool({
connectionString: process.env.PG_URL,
max: Number(process.env.DB_MAX_LINK || 5),
keepAlive: true,
idleTimeoutMillis: 60000,
connectionTimeoutMillis: 20000
});
global.pgClient.on('error', (err) => {
console.log(err);
global.pgClient?.end();
global.pgClient = null;
connectPg();
});
try {
await global.pgClient.connect();
console.log('pg connected');
return global.pgClient;
} catch (error) {
global.pgClient = null;
return connectPg();
}
};
type WhereProps = (string | [string, string | number])[];
type GetProps = {
fields?: string[];
where?: WhereProps;
order?: { field: string; mode: 'DESC' | 'ASC' | string }[];
limit?: number;
offset?: number;
};
type DeleteProps = {
where: WhereProps;
};
type ValuesProps = { key: string; value?: string | number }[];
type UpdateProps = {
values: ValuesProps;
where: WhereProps;
};
type InsertProps = {
values: ValuesProps[];
};
class PgClass {
private getWhereStr(where?: WhereProps) {
return where
? `WHERE ${where
.map((item) => {
if (typeof item === 'string') {
return item;
}
const val = typeof item[1] === 'number' ? item[1] : `'${String(item[1])}'`;
return `${item[0]}=${val}`;
})
.join(' ')}`
: '';
}
private getUpdateValStr(values: ValuesProps) {
return values
.map((item) => {
const val =
typeof item.value === 'number'
? item.value
: `'${String(item.value).replace(/\'/g, '"')}'`;
return `${item.key}=${val}`;
})
.join(',');
}
private getInsertValStr(values: ValuesProps[]) {
return values
.map(
(items) =>
`(${items
.map((item) =>
typeof item.value === 'number'
? item.value
: `'${String(item.value).replace(/\'/g, '"')}'`
)
.join(',')})`
)
.join(',');
}
async select<T extends QueryResultRow = any>(table: string, props: GetProps) {
const sql = `SELECT ${
!props.fields || props.fields?.length === 0 ? '*' : props.fields?.join(',')
}
FROM ${table}
${this.getWhereStr(props.where)}
${
props.order
? `ORDER BY ${props.order.map((item) => `${item.field} ${item.mode}`).join(',')}`
: ''
}
LIMIT ${props.limit || 10} OFFSET ${props.offset || 0}
`;
const pg = await connectPg();
return pg.query<T>(sql);
}
async count(table: string, props: GetProps) {
const sql = `SELECT COUNT(${props?.fields?.[0] || '*'})
FROM ${table}
${this.getWhereStr(props.where)}
`;
const pg = await connectPg();
return pg.query(sql).then((res) => Number(res.rows[0]?.count || 0));
}
async delete(table: string, props: DeleteProps) {
const sql = `DELETE FROM ${table} ${this.getWhereStr(props.where)}`;
const pg = await connectPg();
return pg.query(sql);
}
async update(table: string, props: UpdateProps) {
if (props.values.length === 0) {
return {
rowCount: 0
};
}
const sql = `UPDATE ${table} SET ${this.getUpdateValStr(props.values)} ${this.getWhereStr(
props.where
)}`;
const pg = await connectPg();
return pg.query(sql);
}
async insert(table: string, props: InsertProps) {
if (props.values.length === 0) {
return {
rowCount: 0,
rows: []
};
}
const fields = props.values[0].map((item) => item.key).join(',');
const sql = `INSERT INTO ${table} (${fields}) VALUES ${this.getInsertValStr(
props.values
)} RETURNING id`;
const pg = await connectPg();
return pg.query<{ id: string }>(sql);
}
async query<T extends QueryResultRow = any>(sql: string) {
const pg = await connectPg();
return pg.query<T>(sql);
}
}
export async function initPg() {
try {
await connectPg();
await PgClient.query(`
CREATE EXTENSION IF NOT EXISTS vector;
CREATE TABLE IF NOT EXISTS ${PgDatasetTableName} (
id BIGSERIAL PRIMARY KEY,
vector VECTOR(1536) NOT NULL,
team_id VARCHAR(50) NOT NULL,
tmb_id VARCHAR(50) NOT NULL,
dataset_id VARCHAR(50) NOT NULL,
collection_id VARCHAR(50) NOT NULL,
q TEXT NOT NULL,
a TEXT
);
CREATE INDEX IF NOT EXISTS vector_index ON ${PgDatasetTableName} USING hnsw (vector vector_ip_ops) WITH (m = 24, ef_construction = 64);
`);
console.log('init pg successful');
} catch (error) {
console.log('init pg error', error);
}
}
export const PgClient = new PgClass();
export const Pg = global.pgClient;

5
packages/service/common/pg/type.d.ts vendored Normal file
View File

@@ -0,0 +1,5 @@
import type { Pool } from 'pg';
declare global {
var pgClient: Pool | null;
}

View File

@@ -0,0 +1,6 @@
export enum sseResponseEventEnum {
error = 'error',
answer = 'answer',
moduleStatus = 'moduleStatus',
appStreamResponse = 'appStreamResponse' // sse response request
}

View File

@@ -1,4 +1,98 @@
import type { NextApiResponse } from 'next';
import { sseResponseEventEnum } from './constant';
import { proxyError, ERROR_RESPONSE, ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { addLog } from '../mongo/controller';
import { clearCookie } from '../../support/permission/controller';
export interface ResponseType<T = any> {
code: number;
message: string;
data: T;
}
export const jsonRes = <T = any>(
res: NextApiResponse,
props?: {
code?: number;
message?: string;
data?: T;
error?: any;
}
) => {
const { code = 200, message = '', data = null, error } = props || {};
const errResponseKey = typeof error === 'string' ? error : error?.message;
// Specified error
if (ERROR_RESPONSE[errResponseKey]) {
// login is expired
if (errResponseKey === ERROR_ENUM.unAuthorization) {
clearCookie(res);
}
return res.json(ERROR_RESPONSE[errResponseKey]);
}
// another error
let msg = '';
if ((code < 200 || code >= 400) && !message) {
msg = error?.response?.statusText || error?.message || '请求错误';
if (typeof error === 'string') {
msg = error;
} else if (proxyError[error?.code]) {
msg = '网络连接异常';
} else if (error?.response?.data?.error?.message) {
msg = error?.response?.data?.error?.message;
} else if (error?.error?.message) {
msg = error?.error?.message;
}
addLog.error(`response error: ${msg}`, error);
}
res.status(code).json({
code,
statusText: '',
message: message || msg,
data: data !== undefined ? data : null
});
};
export const sseErrRes = (res: NextApiResponse, error: any) => {
const errResponseKey = typeof error === 'string' ? error : error?.message;
// Specified error
if (ERROR_RESPONSE[errResponseKey]) {
// login is expired
if (errResponseKey === ERROR_ENUM.unAuthorization) {
clearCookie(res);
}
return responseWrite({
res,
event: sseResponseEventEnum.error,
data: JSON.stringify(ERROR_RESPONSE[errResponseKey])
});
}
let msg = error?.response?.statusText || error?.message || '请求错误';
if (typeof error === 'string') {
msg = error;
} else if (proxyError[error?.code]) {
msg = '网络连接异常';
} else if (error?.response?.data?.error?.message) {
msg = error?.response?.data?.error?.message;
} else if (error?.error?.message) {
msg = error?.error?.message;
}
addLog.error(`sse error: ${msg}`, error);
responseWrite({
res,
event: sseResponseEventEnum.error,
data: JSON.stringify({ message: msg })
});
};
export function responseWriteController({
res,

View File

@@ -0,0 +1,26 @@
import { Text2SpeechProps } from '@fastgpt/global/core/ai/speech/api';
import { getAIApi } from '../config';
import { defaultAudioSpeechModels } from '../../../../global/core/ai/model';
import { Text2SpeechVoiceEnum } from '@fastgpt/global/core/ai/speech/constant';
export async function text2Speech({
model = defaultAudioSpeechModels[0].model,
voice = Text2SpeechVoiceEnum.alloy,
input,
speed = 1
}: Text2SpeechProps) {
const ai = getAIApi();
const mp3 = await ai.audio.speech.create({
model,
voice,
input,
response_format: 'mp3',
speed
});
const buffer = Buffer.from(await mp3.arrayBuffer());
return {
model,
voice,
tts: buffer
};
}

View File

@@ -1,5 +1,5 @@
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
import OpenAI from 'openai';
import OpenAI from '@fastgpt/global/core/ai';
export const openaiBaseUrl = process.env.OPENAI_BASE_URL || 'https://api.openai.com/v1';
export const baseUrl = process.env.ONEAPI_URL || openaiBaseUrl;

View File

@@ -1,4 +1,4 @@
import type { ChatCompletionRequestMessage } from '@fastgpt/global/core/ai/type.d';
import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d';
import { getAIApi } from '../config';
export const Prompt_QuestionGuide = `我不太清楚问你什么问题,请帮我生成 3 个问题引导我继续提问。问题的长度应小于20个字符按 JSON 格式返回: ["问题1", "问题2", "问题3"]`;
@@ -7,7 +7,7 @@ export async function createQuestionGuide({
messages,
model
}: {
messages: ChatCompletionRequestMessage[];
messages: ChatMessageItemType[];
model: string;
}) {
const ai = getAIApi(undefined, 48000);

View File

@@ -0,0 +1,70 @@
import { connectionMongo, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant';
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
export const appCollectionName = 'apps';
const AppSchema = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
name: {
type: String,
required: true
},
type: {
type: String,
default: 'advanced',
enum: ['basic', 'advanced']
},
avatar: {
type: String,
default: '/icon/logo.svg'
},
intro: {
type: String,
default: ''
},
updateTime: {
type: Date,
default: () => new Date()
},
modules: {
type: Array,
default: []
},
inited: {
type: Boolean
},
permission: {
type: String,
enum: Object.keys(PermissionTypeMap),
default: PermissionTypeEnum.private
}
});
try {
AppSchema.index({ updateTime: -1 });
AppSchema.index({ 'share.collection': -1 });
} catch (error) {
console.log(error);
}
export const MongoApp: Model<AppType> =
models[appCollectionName] || model(appCollectionName, AppSchema);

View File

@@ -0,0 +1,88 @@
import { connectionMongo, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type';
import { ChatRoleMap, TaskResponseKeyEnum } from '@fastgpt/global/core/chat/constants';
import { customAlphabet } from 'nanoid';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
import { appCollectionName } from '../app/schema';
import { userCollectionName } from '../../support/user/schema';
const ChatItemSchema = new Schema({
dataId: {
type: String,
require: true,
default: () => nanoid()
},
appId: {
type: Schema.Types.ObjectId,
ref: appCollectionName,
required: true
},
chatId: {
type: String,
require: true
},
userId: {
type: Schema.Types.ObjectId,
ref: userCollectionName
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
time: {
type: Date,
default: () => new Date()
},
obj: {
type: String,
required: true,
enum: Object.keys(ChatRoleMap)
},
value: {
type: String,
default: ''
},
userFeedback: {
type: String
},
adminFeedback: {
type: {
datasetId: String,
collectionId: String,
dataId: String,
q: String,
a: String
}
},
[TaskResponseKeyEnum.responseData]: {
type: Array,
default: []
},
tts: {
type: Buffer
}
});
try {
ChatItemSchema.index({ time: -1 });
ChatItemSchema.index({ userId: 1 });
ChatItemSchema.index({ appId: 1 });
ChatItemSchema.index({ chatId: 1 });
ChatItemSchema.index({ userFeedback: 1 });
} catch (error) {
console.log(error);
}
export const MongoChatItem: Model<ChatItemType> =
models['chatItem'] || model('chatItem', ChatItemSchema);

View File

@@ -0,0 +1,103 @@
import { connectionMongo, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { ChatSchema as ChatType } from '@fastgpt/global/core/chat/type.d';
import {
ChatRoleMap,
ChatSourceMap,
TaskResponseKeyEnum
} from '@fastgpt/global/core/chat/constants';
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
import { appCollectionName } from '../app/schema';
export const chatCollectionName = 'chat';
const ChatSchema = new Schema({
chatId: {
type: String,
require: true
},
userId: {
type: Schema.Types.ObjectId,
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
appId: {
type: Schema.Types.ObjectId,
ref: appCollectionName,
required: true
},
updateTime: {
type: Date,
default: () => new Date()
},
title: {
type: String,
default: '历史记录'
},
customTitle: {
type: String,
default: ''
},
top: {
type: Boolean
},
variables: {
type: Object,
default: {}
},
source: {
type: String,
enum: Object.keys(ChatSourceMap),
required: true
},
shareId: {
type: String
},
isInit: {
type: Boolean,
default: false
},
content: {
type: [
{
obj: {
type: String,
required: true,
enum: Object.keys(ChatRoleMap)
},
value: {
type: String,
default: ''
},
[TaskResponseKeyEnum.responseData]: {
type: Array,
default: []
}
}
],
default: []
}
});
try {
ChatSchema.index({ userId: 1 });
ChatSchema.index({ updateTime: -1 });
ChatSchema.index({ appId: 1 });
} catch (error) {
console.log(error);
}
export const MongoChat: Model<ChatType> =
models[chatCollectionName] || model(chatCollectionName, ChatSchema);

View File

@@ -1,26 +0,0 @@
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { MongoDatasetCollection } from './collection/schema';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
export async function authCollection({
collectionId,
userId
}: {
collectionId: string;
userId: string;
}) {
const collection = await MongoDatasetCollection.findOne({
_id: collectionId,
userId
})
.populate('datasetId')
.lean();
if (collection) {
return {
...collection,
dataset: collection.datasetId as unknown as DatasetSchemaType
};
}
return Promise.reject(ERROR_ENUM.unAuthDataset);
}

View File

@@ -3,6 +3,10 @@ const { Schema, model, models } = connectionMongo;
import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant';
import { DatasetCollectionName } from '../schema';
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
export const DatasetColCollectionName = 'dataset.collections';
@@ -13,8 +17,18 @@ const DatasetCollectionSchema = new Schema({
default: null
},
userId: {
// abandoned
type: Schema.Types.ObjectId,
ref: 'user',
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
datasetId: {

View File

@@ -31,18 +31,16 @@ export async function findCollectionAndChild(id: string, fields = '_id parentId
}
export async function getDatasetCollectionPaths({
parentId = '',
userId
parentId = ''
}: {
parentId?: string;
userId: string;
}): Promise<ParentTreePathItemType[]> {
async function find(parentId?: string): Promise<ParentTreePathItemType[]> {
if (!parentId) {
return [];
}
const parent = await MongoDatasetCollection.findOne({ _id: parentId, userId }, 'name parentId');
const parent = await MongoDatasetCollection.findOne({ _id: parentId }, 'name parentId');
if (!parent) return [];

View File

@@ -0,0 +1,12 @@
import { CollectionWithDatasetType } from '@fastgpt/global/core/dataset/type';
import { MongoDatasetCollection } from './collection/schema';
export async function getCollectionWithDataset(collectionId: string) {
const data = (
await MongoDatasetCollection.findById(collectionId).populate('datasetId')
)?.toJSON() as CollectionWithDatasetType;
if (!data) {
return Promise.reject('Collection is not exist');
}
return data;
}

View File

@@ -0,0 +1,9 @@
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { getGFSCollection } from '../../../common/file/gridfs/controller';
export async function delDatasetFiles({ datasetId }: { datasetId: string }) {
const db = getGFSCollection(BucketNameEnum.dataset);
await db.deleteMany({
'metadata.datasetId': String(datasetId)
});
}

View File

@@ -2,6 +2,11 @@ import { connectionMongo, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constant';
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant';
export const DatasetCollectionName = 'datasets';
@@ -12,8 +17,18 @@ const DatasetSchema = new Schema({
default: null
},
userId: {
//abandon
type: Schema.Types.ObjectId,
ref: 'user',
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
updateTime: {
@@ -41,7 +56,16 @@ const DatasetSchema = new Schema({
},
tags: {
type: [String],
default: []
default: [],
set(val: string | string[]) {
if (Array.isArray(val)) return val;
return val.split(' ').filter((item) => item);
}
},
permission: {
type: String,
enum: Object.keys(PermissionTypeMap),
default: PermissionTypeEnum.private
}
});

View File

@@ -5,13 +5,27 @@ import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
import { DatasetColCollectionName } from '../collection/schema';
import { DatasetCollectionName } from '../schema';
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
export const DatasetTrainingCollectionName = 'dataset.trainings';
const TrainingDataSchema = new Schema({
userId: {
// abandon
type: Schema.Types.ObjectId,
ref: 'user',
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
datasetId: {

View File

@@ -1,38 +1,15 @@
import { CreateOnePluginParams, UpdatePluginParams } from '@fastgpt/global/core/plugin/controller';
import { MongoPlugin } from './schema';
import { FlowModuleTemplateType } from '@fastgpt/global/core/module/type';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { formatPluginIOModules } from '@fastgpt/global/core/module/utils';
export async function createOnePlugin(data: CreateOnePluginParams & { userId: string }) {
const { _id } = await MongoPlugin.create(data);
return _id;
}
export async function updateOnePlugin({
id,
userId,
...data
}: UpdatePluginParams & { userId: string }) {
await MongoPlugin.findOneAndUpdate({ _id: id, userId }, data);
}
export async function deleteOnePlugin({ id, userId }: { id: string; userId: string }) {
await MongoPlugin.findOneAndDelete({ _id: id, userId });
}
export async function getUserPlugins({ userId }: { userId: string }) {
return MongoPlugin.find({ userId }, 'name avatar intro');
}
export async function getOnePluginDetail({ id, userId }: { userId: string; id: string }) {
return MongoPlugin.findOne({ _id: id, userId });
}
/* plugin templates */
export async function getUserPlugins2Templates({
userId
teamId
}: {
userId: string;
teamId: string;
}): Promise<FlowModuleTemplateType[]> {
const plugins = await MongoPlugin.find({ userId }).lean();
const plugins = await MongoPlugin.find({ teamId }).lean();
return plugins.map((plugin) => ({
id: String(plugin._id),
@@ -47,8 +24,8 @@ export async function getUserPlugins2Templates({
}));
}
/* one plugin 2 module detail */
export async function getPluginModuleDetail({ id, userId }: { userId: string; id: string }) {
const plugin = await getOnePluginDetail({ id, userId });
export async function getPluginModuleDetail({ id }: { id: string }) {
const plugin = await MongoPlugin.findById(id);
if (!plugin) return Promise.reject('plugin not found');
return {
id: String(plugin._id),

View File

@@ -1,13 +1,26 @@
import { connectionMongo, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import type { PluginItemSchema } from '@fastgpt/global/core/plugin/type.d';
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
export const ModuleCollectionName = 'plugins';
const PluginSchema = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
name: {

View File

@@ -13,10 +13,12 @@
"winston-mongodb": "^5.1.1",
"tunnel": "^0.0.6",
"encoding": "^0.1.13",
"openai": "^4.12.4"
"pg": "^8.10.0",
"nanoid": "^4.0.1"
},
"devDependencies": {
"@types/tunnel": "^0.0.4",
"@types/pg": "^8.6.6",
"@types/node": "^20.8.5",
"@types/cookie": "^0.5.2",
"@types/jsonwebtoken": "^9.0.3"

View File

@@ -10,13 +10,11 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
if (!apikey) {
return Promise.reject(ERROR_ENUM.unAuthApiKey);
}
try {
const openApi = await MongoOpenApi.findOne({ apiKey: apikey });
if (!openApi) {
return Promise.reject(ERROR_ENUM.unAuthApiKey);
}
const userId = String(openApi.userId);
// auth limit
if (global.feConfigs?.isPlus) {
@@ -25,7 +23,13 @@ export async function authOpenApiKey({ apikey }: { apikey: string }) {
updateApiKeyUsedTime(openApi._id);
return { apikey, userId, appId: openApi.appId };
return {
apikey,
userId: String(openApi.userId),
teamId: String(openApi.teamId),
tmbId: String(openApi.tmbId),
appId: openApi.appId || ''
};
} catch (error) {
return Promise.reject(error);
}

View File

@@ -1,14 +1,27 @@
import { connectionMongo, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
import { PRICE_SCALE } from '@fastgpt/global/common/bill/constants';
import { formatPrice } from '@fastgpt/global/common/bill/tools';
import { PRICE_SCALE } from '@fastgpt/global/support/wallet/bill/constants';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
const OpenApiSchema = new Schema(
{
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
apiKey: {

View File

@@ -1,68 +0,0 @@
import { AuthUserTypeEnum, authBalanceByUid } from '../user/auth';
import { MongoOutLink } from './schema';
import { POST } from '../../common/api/plusRequest';
import { OutLinkSchema } from '@fastgpt/global/support/outLink/type';
export type AuthLinkProps = { ip?: string | null; authToken?: string; question: string };
export type AuthLinkLimitProps = AuthLinkProps & { outLink: OutLinkSchema };
export async function authOutLinkChat({
shareId,
ip,
authToken,
question
}: AuthLinkProps & {
shareId: string;
}) {
// get outLink
const outLink = await MongoOutLink.findOne({
shareId
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
const uid = String(outLink.userId);
const [user] = await Promise.all([
authBalanceByUid(uid), // authBalance
...(global.feConfigs?.isPlus ? [authOutLinkLimit({ outLink, ip, authToken, question })] : []) // limit auth
]);
return {
user,
userId: String(outLink.userId),
appId: String(outLink.appId),
authType: AuthUserTypeEnum.token,
responseDetail: outLink.responseDetail
};
}
export function authOutLinkLimit(data: AuthLinkLimitProps) {
return POST('/support/outLink/authLimit', data);
}
export async function authOutLinkId({ id }: { id: string }) {
const outLink = await MongoOutLink.findOne({
shareId: id
});
if (!outLink) {
return Promise.reject('分享链接无效');
}
return {
userId: String(outLink.userId)
};
}
export type AuthShareChatInitProps = {
authToken?: string;
tokenUrl?: string;
};
export function authShareChatInit(data: AuthShareChatInitProps) {
if (!global.feConfigs?.isPlus) return;
return POST('/support/outLink/authShareChatInit', data);
}

View File

@@ -2,6 +2,10 @@ import { connectionMongo, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { OutLinkSchema as SchemaType } from '@fastgpt/global/support/outLink/type';
import { OutLinkTypeEnum } from '@fastgpt/global/support/outLink/constant';
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
const OutLinkSchema = new Schema({
shareId: {
@@ -10,7 +14,16 @@ const OutLinkSchema = new Schema({
},
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
appId: {

View File

@@ -0,0 +1,72 @@
import { MongoApp } from '../../../core/app/schema';
import { AppDetailType } from '@fastgpt/global/core/app/type.d';
import { AuthModeType } from '../type';
import { AuthResponseType } from '@fastgpt/global/support/permission/type';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { parseHeaderCert } from '../controller';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { AppErrEnum } from '@fastgpt/global/common/error/code/app';
import { getTeamInfoByTmbId } from '../../user/team/controller';
// 模型使用权校验
export async function authApp({
appId,
per = 'owner',
...props
}: AuthModeType & {
appId: string;
}): Promise<
AuthResponseType & {
teamOwner: boolean;
app: AppDetailType;
}
> {
const result = await parseHeaderCert(props);
const { userId, teamId, tmbId } = result;
const { role } = await getTeamInfoByTmbId({ tmbId });
const { app, isOwner, canWrite } = await (async () => {
// get app
const app = (await MongoApp.findOne({ _id: appId, teamId }))?.toJSON();
if (!app) {
return Promise.reject(AppErrEnum.unAuthApp);
}
const isOwner =
role !== TeamMemberRoleEnum.visitor &&
(String(app.tmbId) === tmbId || role === TeamMemberRoleEnum.owner);
const canWrite =
isOwner ||
(app.permission === PermissionTypeEnum.public && role !== TeamMemberRoleEnum.visitor);
if (per === 'r') {
if (!isOwner && app.permission !== PermissionTypeEnum.public) {
return Promise.reject(AppErrEnum.unAuthApp);
}
}
if (per === 'w' && !canWrite) {
return Promise.reject(AppErrEnum.unAuthApp);
}
if (per === 'owner' && !isOwner) {
return Promise.reject(AppErrEnum.unAuthApp);
}
return {
app: {
...app,
isOwner,
canWrite
},
isOwner,
canWrite
};
})();
return {
...result,
app,
isOwner,
canWrite,
teamOwner: role === TeamMemberRoleEnum.owner
};
}

View File

@@ -0,0 +1,67 @@
import { AuthResponseType } from '@fastgpt/global/support/permission/type';
import { AuthModeType } from '../type';
import type { ChatSchema, ChatWithAppSchema } from '@fastgpt/global/core/chat/type';
import { parseHeaderCert } from '../controller';
import { MongoChat } from '../../../core/chat/chatSchema';
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
import { getTeamInfoByTmbId } from '../../user/team/controller';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
export async function authChat({
chatId,
per = 'owner',
...props
}: AuthModeType & {
chatId: string;
}): Promise<
AuthResponseType & {
chat: ChatSchema;
}
> {
const { userId, teamId, tmbId } = await parseHeaderCert(props);
const { role } = await getTeamInfoByTmbId({ tmbId });
const { chat, isOwner, canWrite } = await (async () => {
// get chat
const chat = (
await MongoChat.findOne({ chatId, teamId }).populate('appId')
)?.toJSON() as ChatWithAppSchema;
if (!chat) {
return Promise.reject('Chat is not exists');
}
const isOwner = role === TeamMemberRoleEnum.owner || String(chat.tmbId) === tmbId;
const canWrite = isOwner;
if (
per === 'r' &&
role !== TeamMemberRoleEnum.owner &&
chat.appId.permission !== PermissionTypeEnum.public
) {
return Promise.reject(ChatErrEnum.unAuthChat);
}
if (per === 'w' && !canWrite) {
return Promise.reject(ChatErrEnum.unAuthChat);
}
if (per === 'owner' && !isOwner) {
return Promise.reject(ChatErrEnum.unAuthChat);
}
return {
chat,
isOwner,
canWrite
};
})();
return {
userId,
teamId,
tmbId,
chat,
isOwner,
canWrite
};
}

View File

@@ -0,0 +1,12 @@
import { parseHeaderCert } from '../controller';
import { AuthModeType } from '../type';
export const authCert = async (props: AuthModeType) => {
const result = await parseHeaderCert(props);
return {
...result,
isOwner: true,
canWrite: true
};
};

View File

@@ -0,0 +1,181 @@
import { AuthModeType } from '../type';
import { parseHeaderCert } from '../controller';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { MongoDataset } from '../../../core/dataset/schema';
import { getCollectionWithDataset } from '../../../core/dataset/controller';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { AuthResponseType } from '@fastgpt/global/support/permission/type';
import {
CollectionWithDatasetType,
DatasetFileSchema,
DatasetSchemaType
} from '@fastgpt/global/core/dataset/type';
import { getFileById } from '../../../common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { getTeamInfoByTmbId } from '../../user/team/controller';
export async function authDataset({
datasetId,
per = 'owner',
...props
}: AuthModeType & {
datasetId: string;
}): Promise<
AuthResponseType & {
dataset: DatasetSchemaType;
}
> {
const result = await parseHeaderCert(props);
const { userId, teamId, tmbId } = result;
const { role } = await getTeamInfoByTmbId({ tmbId });
const { dataset, isOwner, canWrite } = await (async () => {
const dataset = (await MongoDataset.findOne({ _id: datasetId, teamId }))?.toJSON();
if (!dataset) {
return Promise.reject(DatasetErrEnum.unAuthDataset);
}
const isOwner =
role !== TeamMemberRoleEnum.visitor &&
(String(dataset.tmbId) === tmbId || role === TeamMemberRoleEnum.owner);
const canWrite =
isOwner ||
(role !== TeamMemberRoleEnum.visitor && dataset.permission === PermissionTypeEnum.public);
if (per === 'r') {
if (!isOwner && dataset.permission !== PermissionTypeEnum.public) {
return Promise.reject(DatasetErrEnum.unAuthDataset);
}
}
if (per === 'w' && !canWrite) {
return Promise.reject(DatasetErrEnum.unAuthDataset);
}
if (per === 'owner' && !isOwner) {
return Promise.reject(DatasetErrEnum.unAuthDataset);
}
return { dataset, isOwner, canWrite };
})();
return {
...result,
dataset,
isOwner,
canWrite
};
}
/*
Read: in team and dataset permission is public
Write: in team, not visitor and dataset permission is public
*/
export async function authDatasetCollection({
collectionId,
per = 'owner',
...props
}: AuthModeType & {
collectionId: string;
}): Promise<
AuthResponseType & {
collection: CollectionWithDatasetType;
}
> {
const { userId, teamId, tmbId } = await parseHeaderCert(props);
const { role } = await getTeamInfoByTmbId({ tmbId });
const { collection, isOwner, canWrite } = await (async () => {
const collection = await getCollectionWithDataset(collectionId);
if (!collection || String(collection.teamId) !== teamId) {
return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
}
const isOwner =
String(collection.datasetId.tmbId) === tmbId || role === TeamMemberRoleEnum.owner;
const canWrite =
isOwner ||
(role !== TeamMemberRoleEnum.visitor &&
collection.datasetId.permission === PermissionTypeEnum.public);
if (per === 'r') {
if (!isOwner && collection.datasetId.permission !== PermissionTypeEnum.public) {
return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
}
}
if (per === 'w' && !canWrite) {
return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
}
if (per === 'owner' && !isOwner) {
return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
}
return {
collection,
isOwner,
canWrite
};
})();
return {
userId,
teamId,
tmbId,
collection,
isOwner,
canWrite
};
}
export async function authDatasetFile({
fileId,
per = 'owner',
...props
}: AuthModeType & {
fileId: string;
}): Promise<
AuthResponseType & {
file: DatasetFileSchema;
}
> {
const { userId, teamId, tmbId } = await parseHeaderCert(props);
const { role } = await getTeamInfoByTmbId({ tmbId });
const file = await getFileById({ bucketName: BucketNameEnum.dataset, fileId });
if (file.metadata.teamId !== teamId) {
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
}
const { dataset } = await authDataset({
...props,
datasetId: file.metadata.datasetId,
per
});
const isOwner =
role !== TeamMemberRoleEnum.visitor &&
(String(dataset.tmbId) === tmbId || role === TeamMemberRoleEnum.owner);
const canWrite =
isOwner ||
(role !== TeamMemberRoleEnum.visitor && dataset.permission === PermissionTypeEnum.public);
if (per === 'r' && !isOwner && dataset.permission !== PermissionTypeEnum.public) {
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
}
if (per === 'w' && !canWrite) {
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
}
if (per === 'owner' && !isOwner) {
return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
}
return {
userId,
teamId,
tmbId,
file,
isOwner,
canWrite
};
}

View File

@@ -0,0 +1,61 @@
import { AuthResponseType } from '@fastgpt/global/support/permission/type';
import { AuthModeType } from '../type';
import { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
import { parseHeaderCert } from '../controller';
import { getTeamInfoByTmbId } from '../../user/team/controller';
import { MongoOpenApi } from '../../openapi/schema';
import { OpenApiErrEnum } from '@fastgpt/global/common/error/code/openapi';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
export async function authOpenApiKeyCrud({
id,
per = 'owner',
...props
}: AuthModeType & {
id: string;
}): Promise<
AuthResponseType & {
openapi: OpenApiSchema;
}
> {
const result = await parseHeaderCert(props);
const { tmbId, teamId } = result;
const { role } = await getTeamInfoByTmbId({ tmbId });
const { openapi, isOwner, canWrite } = await (async () => {
const openapi = await MongoOpenApi.findOne({ _id: id, teamId });
if (!openapi) {
throw new Error(OpenApiErrEnum.unExist);
}
const isOwner = String(openapi.tmbId) === tmbId || role === TeamMemberRoleEnum.owner;
const canWrite =
isOwner || (String(openapi.tmbId) === tmbId && role !== TeamMemberRoleEnum.visitor);
if (per === 'r' && !canWrite) {
return Promise.reject(OpenApiErrEnum.unAuth);
}
if (per === 'w' && !canWrite) {
return Promise.reject(OpenApiErrEnum.unAuth);
}
if (per === 'owner' && !isOwner) {
return Promise.reject(OpenApiErrEnum.unAuth);
}
return {
openapi,
isOwner,
canWrite
};
})();
return {
...result,
openapi,
isOwner,
canWrite
};
}

View File

@@ -0,0 +1,98 @@
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { AuthModeType } from '../type';
import { AuthResponseType } from '@fastgpt/global/support/permission/type';
import { AppDetailType } from '@fastgpt/global/core/app/type';
import { OutLinkSchema } from '@fastgpt/global/support/outLink/type';
import { parseHeaderCert } from '../controller';
import { MongoOutLink } from '../../outLink/schema';
import { MongoApp } from '../../../core/app/schema';
import { OutLinkErrEnum } from '@fastgpt/global/common/error/code/outLink';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { AppErrEnum } from '@fastgpt/global/common/error/code/app';
import { getTeamInfoByTmbId } from '../../user/team/controller';
/* crud outlink permission */
export async function authOutLinkCrud({
outLinkId,
per = 'owner',
...props
}: AuthModeType & {
outLinkId: string;
}): Promise<
AuthResponseType & {
app: AppDetailType;
outLink: OutLinkSchema;
}
> {
const result = await parseHeaderCert(props);
const { tmbId, teamId } = result;
const { role } = await getTeamInfoByTmbId({ tmbId });
const { app, outLink, isOwner, canWrite } = await (async () => {
const outLink = await MongoOutLink.findOne({ _id: outLinkId, teamId });
if (!outLink) {
throw new Error(OutLinkErrEnum.unExist);
}
const app = await MongoApp.findById(outLink.appId);
if (!app) {
return Promise.reject(AppErrEnum.unExist);
}
const isOwner = String(outLink.tmbId) === tmbId || role === TeamMemberRoleEnum.owner;
const canWrite =
isOwner ||
(app.permission === PermissionTypeEnum.public && role !== TeamMemberRoleEnum.visitor);
if (per === 'r' && !isOwner && app.permission !== PermissionTypeEnum.public) {
return Promise.reject(OutLinkErrEnum.unAuthLink);
}
if (per === 'w' && !canWrite) {
return Promise.reject(OutLinkErrEnum.unAuthLink);
}
if (per === 'owner' && !isOwner) {
return Promise.reject(OutLinkErrEnum.unAuthLink);
}
return {
app: {
...app,
isOwner: String(app.tmbId) === tmbId,
canWrite
},
outLink,
isOwner,
canWrite
};
})();
return {
...result,
app,
outLink,
isOwner,
canWrite
};
}
export async function authOutLinkValid({ shareId }: { shareId?: string }) {
const shareChat = await MongoOutLink.findOne({ shareId });
if (!shareChat) {
return Promise.reject(OutLinkErrEnum.linkUnInvalid);
}
const app = await MongoApp.findById(shareChat.appId);
if (!app) {
return Promise.reject(AppErrEnum.unExist);
}
return {
app,
shareChat
};
}

View File

@@ -0,0 +1,56 @@
import { AuthResponseType } from '@fastgpt/global/support/permission/type';
import { AuthModeType } from '../type';
import { parseHeaderCert } from '../controller';
import { getTeamInfoByTmbId } from '../../user/team/controller';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { MongoPlugin } from '../../../core/plugin/schema';
import { PluginErrEnum } from '@fastgpt/global/common/error/code/plugin';
import { PluginItemSchema } from '@fastgpt/global/core/plugin/type';
export async function authPluginCrud({
id,
per = 'owner',
...props
}: AuthModeType & {
id: string;
}): Promise<
AuthResponseType & {
plugin: PluginItemSchema;
}
> {
const result = await parseHeaderCert(props);
const { tmbId, teamId } = result;
const { role } = await getTeamInfoByTmbId({ tmbId });
const { plugin, isOwner, canWrite } = await (async () => {
const plugin = await MongoPlugin.findOne({ _id: id, teamId });
if (!plugin) {
throw new Error(PluginErrEnum.unExist);
}
const isOwner = String(plugin.tmbId) === tmbId || role === TeamMemberRoleEnum.owner;
const canWrite = isOwner;
if (per === 'w' && !canWrite) {
return Promise.reject(PluginErrEnum.unAuth);
}
if (per === 'owner' && !isOwner) {
return Promise.reject(PluginErrEnum.unAuth);
}
return {
plugin,
isOwner,
canWrite
};
})();
return {
...result,
plugin,
isOwner,
canWrite
};
}

View File

@@ -0,0 +1,52 @@
import { AuthResponseType } from '@fastgpt/global/support/permission/type';
import { AuthModeType } from '../type';
import { TeamItemType } from '@fastgpt/global/support/user/team/type';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { parseHeaderCert } from '../controller';
import { getTeamInfoByTmbId } from '../../user/team/controller';
import { UserErrEnum } from '../../../../global/common/error/code/user';
export async function authUserNotVisitor(props: AuthModeType): Promise<
AuthResponseType & {
team: TeamItemType;
role: `${TeamMemberRoleEnum}`;
}
> {
const { userId, teamId, tmbId } = await parseHeaderCert(props);
const team = await getTeamInfoByTmbId({ tmbId });
if (team.role === TeamMemberRoleEnum.visitor) {
return Promise.reject(UserErrEnum.binVisitor);
}
return {
userId,
teamId,
tmbId,
team,
role: team.role,
isOwner: team.role === TeamMemberRoleEnum.owner, // teamOwner
canWrite: true
};
}
/* auth user role */
export async function authUserRole(props: AuthModeType): Promise<
AuthResponseType & {
role: `${TeamMemberRoleEnum}`;
teamOwner: boolean;
}
> {
const { userId, teamId, tmbId } = await parseHeaderCert(props);
const { role: userRole, canWrite } = await getTeamInfoByTmbId({ tmbId });
return {
userId,
teamId,
tmbId,
isOwner: true,
role: userRole,
teamOwner: userRole === TeamMemberRoleEnum.owner,
canWrite
};
}

View File

@@ -0,0 +1,241 @@
import Cookie from 'cookie';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import jwt from 'jsonwebtoken';
import { NextApiResponse } from 'next';
import type { AuthModeType, ReqHeaderAuthType } from './type.d';
import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
import { authOpenApiKey } from '../openapi/auth';
import { FileTokenQuery } from '@fastgpt/global/common/file/type';
/* create token */
export function createJWT(user: { _id?: string; team?: { teamId?: string; tmbId: string } }) {
const key = process.env.TOKEN_KEY as string;
const token = jwt.sign(
{
userId: String(user._id),
teamId: String(user.team?.teamId),
tmbId: String(user.team?.tmbId),
exp: Math.floor(Date.now() / 1000) + 60 * 60 * 24 * 7
},
key
);
return token;
}
// auth token
export function authJWT(token: string) {
return new Promise<{
userId: string;
teamId: string;
tmbId: string;
}>((resolve, reject) => {
const key = process.env.TOKEN_KEY as string;
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded?.userId) {
reject(ERROR_ENUM.unAuthorization);
return;
}
resolve({
userId: decoded.userId,
teamId: decoded.teamId || '',
tmbId: decoded.tmbId
});
});
});
}
export async function parseHeaderCert({
req,
authToken = false,
authRoot = false,
authApiKey = false
}: AuthModeType) {
// parse jwt
async function authCookieToken(cookie?: string, token?: string) {
// 获取 cookie
const cookies = Cookie.parse(cookie || '');
const cookieToken = cookies.token || token;
if (!cookieToken) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return await authJWT(cookieToken);
}
// from authorization get apikey
async function parseAuthorization(authorization?: string) {
if (!authorization) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// Bearer fastgpt-xxxx-appId
const auth = authorization.split(' ')[1];
if (!auth) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
const { apikey, appId: authorizationAppid = '' } = await (async () => {
const arr = auth.split('-');
// abandon
if (arr.length === 3) {
return {
apikey: `${arr[0]}-${arr[1]}`,
appId: arr[2]
};
}
if (arr.length === 2) {
return {
apikey: auth
};
}
return Promise.reject(ERROR_ENUM.unAuthorization);
})();
// auth apikey
const { userId, teamId, tmbId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
return {
uid: userId,
teamId,
tmbId,
apikey,
appId: apiKeyAppId || authorizationAppid
};
}
// root user
async function parseRootKey(rootKey?: string, userId = '') {
if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return userId;
}
const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers ||
{}) as ReqHeaderAuthType;
const { uid, teamId, tmbId, appId, openApiKey, authType } = await (async () => {
if (authToken && (cookie || token)) {
// user token(from fastgpt web)
const res = await authCookieToken(cookie, token);
return {
uid: res.userId,
teamId: res.teamId,
tmbId: res.tmbId,
appId: '',
openApiKey: '',
authType: AuthUserTypeEnum.token
};
}
if (authRoot && rootkey) {
// root user
return {
uid: await parseRootKey(rootkey, userid),
teamId: '',
tmbId: '',
appId: '',
openApiKey: '',
authType: AuthUserTypeEnum.root
};
}
if (authApiKey && apikey) {
// apikey
const parseResult = await authOpenApiKey({ apikey });
return {
uid: parseResult.userId,
teamId: parseResult.teamId,
tmbId: parseResult.tmbId,
appId: parseResult.appId,
openApiKey: parseResult.apikey,
authType: AuthUserTypeEnum.apikey
};
}
if (authApiKey && authorization) {
// apikey from authorization
const authResponse = await parseAuthorization(authorization);
return {
uid: authResponse.uid,
teamId: authResponse.teamId,
tmbId: authResponse.tmbId,
appId: authResponse.appId,
openApiKey: authResponse.apikey,
authType: AuthUserTypeEnum.apikey
};
}
return {
uid: '',
teamId: '',
tmbId: '',
appId: '',
openApiKey: '',
authType: AuthUserTypeEnum.token
};
})();
// not rootUser and no uid, reject request
if (!rootkey && !uid && !teamId && !tmbId) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return {
userId: String(uid),
teamId: String(teamId),
tmbId: String(tmbId),
appId,
authType,
apikey: openApiKey
};
}
/* set cookie */
export const setCookie = (res: NextApiResponse, token: string) => {
res.setHeader(
'Set-Cookie',
`token=${token}; Path=/; HttpOnly; Max-Age=604800; Samesite=None; Secure;`
);
};
/* clear cookie */
export const clearCookie = (res: NextApiResponse) => {
res.setHeader('Set-Cookie', 'token=; Path=/; Max-Age=0');
};
/* file permission */
export const createFileToken = (data: FileTokenQuery) => {
if (!process.env.FILE_TOKEN_KEY) {
return Promise.reject('System unset FILE_TOKEN_KEY');
}
const expiredTime = Math.floor(Date.now() / 1000) + 60 * 30;
const key = process.env.FILE_TOKEN_KEY as string;
const token = jwt.sign(
{
...data,
exp: expiredTime
},
key
);
return Promise.resolve(token);
};
export const authFileToken = (token?: string) =>
new Promise<FileTokenQuery>((resolve, reject) => {
if (!token) {
return reject(ERROR_ENUM.unAuthFile);
}
const key = process.env.FILE_TOKEN_KEY as string;
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded.bucketName || !decoded?.teamId || !decoded?.tmbId || !decoded?.fileId) {
reject(ERROR_ENUM.unAuthFile);
return;
}
resolve({
bucketName: decoded.bucketName,
teamId: decoded.teamId,
tmbId: decoded.tmbId,
fileId: decoded.fileId
});
});
});

View File

@@ -0,0 +1,17 @@
import { NextApiRequest } from 'next';
export type ReqHeaderAuthType = {
cookie?: string;
token?: string;
apikey?: string; // abandon
rootkey?: string;
userid?: string;
authorization?: string;
};
export type AuthModeType = {
req: NextApiRequest;
authToken?: boolean;
authRoot?: boolean;
authApiKey?: boolean;
per?: 'r' | 'w' | 'owner';
};

View File

@@ -1,206 +0,0 @@
import type { NextApiResponse, NextApiRequest } from 'next';
import Cookie from 'cookie';
import jwt from 'jsonwebtoken';
import { authOpenApiKey } from '../openapi/auth';
import { authOutLinkId } from '../outLink/auth';
import { MongoUser } from './schema';
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
export enum AuthUserTypeEnum {
token = 'token',
root = 'root',
apikey = 'apikey',
outLink = 'outLink'
}
/* auth balance */
export const authBalanceByUid = async (uid: string) => {
const user = await MongoUser.findById<UserModelSchema>(
uid,
'_id username balance openaiAccount timezone'
);
if (!user) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
if (user.balance <= 0) {
return Promise.reject(ERROR_ENUM.insufficientQuota);
}
return user;
};
/* uniform auth user */
export const authUser = async ({
req,
authToken = false,
authRoot = false,
authApiKey = false,
authBalance = false,
authOutLink
}: {
req: NextApiRequest;
authToken?: boolean;
authRoot?: boolean;
authApiKey?: boolean;
authBalance?: boolean;
authOutLink?: boolean;
}) => {
const authCookieToken = async (cookie?: string, token?: string): Promise<string> => {
// 获取 cookie
const cookies = Cookie.parse(cookie || '');
const cookieToken = cookies.token || token;
if (!cookieToken) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return await authJWT(cookieToken);
};
// from authorization get apikey
const parseAuthorization = async (authorization?: string) => {
if (!authorization) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// Bearer fastgpt-xxxx-appId
const auth = authorization.split(' ')[1];
if (!auth) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
const { apikey, appId: authorizationAppid = '' } = await (async () => {
const arr = auth.split('-');
// abandon
if (arr.length === 3) {
return {
apikey: `${arr[0]}-${arr[1]}`,
appId: arr[2]
};
}
if (arr.length === 2) {
return {
apikey: auth
};
}
return Promise.reject(ERROR_ENUM.unAuthorization);
})();
// auth apikey
const { userId, appId: apiKeyAppId = '' } = await authOpenApiKey({ apikey });
return {
uid: userId,
apikey,
appId: apiKeyAppId || authorizationAppid
};
};
// root user
const parseRootKey = async (rootKey?: string, userId = '') => {
if (!rootKey || !process.env.ROOT_KEY || rootKey !== process.env.ROOT_KEY) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
return userId;
};
const { cookie, token, apikey, rootkey, userid, authorization } = (req.headers || {}) as {
cookie?: string;
token?: string;
apikey?: string;
rootkey?: string; // abandon
userid?: string;
authorization?: string;
};
const { shareId } = (req?.body || {}) as { shareId?: string };
let uid = '';
let appId = '';
let openApiKey = apikey;
let authType: `${AuthUserTypeEnum}` = AuthUserTypeEnum.token;
if (authOutLink && shareId) {
const res = await authOutLinkId({ id: shareId });
uid = res.userId;
authType = AuthUserTypeEnum.outLink;
} else if (authToken && (cookie || token)) {
// user token(from fastgpt web)
uid = await authCookieToken(cookie, token);
authType = AuthUserTypeEnum.token;
} else if (authRoot && rootkey) {
// root user
uid = await parseRootKey(rootkey, userid);
authType = AuthUserTypeEnum.root;
} else if (authApiKey && apikey) {
// apikey
const parseResult = await authOpenApiKey({ apikey });
uid = parseResult.userId;
authType = AuthUserTypeEnum.apikey;
openApiKey = parseResult.apikey;
} else if (authApiKey && authorization) {
// apikey from authorization
const authResponse = await parseAuthorization(authorization);
uid = authResponse.uid;
appId = authResponse.appId;
openApiKey = authResponse.apikey;
authType = AuthUserTypeEnum.apikey;
}
// not rootUser and no uid, reject request
if (!rootkey && !uid) {
return Promise.reject(ERROR_ENUM.unAuthorization);
}
// balance check
const user = await (() => {
if (authBalance) {
return authBalanceByUid(uid);
}
})();
return {
userId: String(uid),
appId,
authType,
user,
apikey: openApiKey
};
};
/* 生成 token */
export function generateToken(userId: string) {
const key = process.env.TOKEN_KEY as string;
const token = jwt.sign(
{
userId,
exp: Math.floor(Date.now() / 1000) + 60 * 60 * 24 * 7
},
key
);
return token;
}
// auth token
export function authJWT(token: string) {
return new Promise<string>((resolve, reject) => {
const key = process.env.TOKEN_KEY as string;
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded?.userId) {
reject(ERROR_ENUM.unAuthorization);
return;
}
resolve(decoded.userId);
});
});
}
/* set cookie */
export const setCookie = (res: NextApiResponse, token: string) => {
res.setHeader(
'Set-Cookie',
`token=${token}; Path=/; HttpOnly; Max-Age=604800; Samesite=None; Secure;`
);
};
/* clear cookie */
export const clearCookie = (res: NextApiResponse) => {
res.setHeader('Set-Cookie', 'token=; Path=/; Max-Age=0');
};

View File

@@ -0,0 +1,11 @@
import { MongoUser } from './schema';
export async function authUserExist({ userId, username }: { userId?: string; username?: string }) {
if (userId) {
return MongoUser.findOne({ _id: userId });
}
if (username) {
return MongoUser.findOne({ username });
}
return null;
}

View File

@@ -1,45 +0,0 @@
import { MongoUserInform } from './schema';
import { MongoUser } from '../schema';
import { InformTypeEnum } from '@fastgpt/global/support/user/constant';
export type SendInformProps = {
type: `${InformTypeEnum}`;
title: string;
content: string;
};
export async function sendInform2AllUser({ type, title, content }: SendInformProps) {
const users = await MongoUser.find({}, '_id');
await MongoUserInform.insertMany(
users.map(({ _id }) => ({
type,
title,
content,
userId: _id
}))
);
}
export async function sendInform2OneUser({
type,
title,
content,
userId
}: SendInformProps & { userId: string }) {
const inform = await MongoUserInform.findOne({
type,
title,
content,
userId,
time: { $gte: new Date(Date.now() - 5 * 60 * 1000) }
});
if (inform) return;
await MongoUserInform.create({
type,
title,
content,
userId
});
}

View File

@@ -1,42 +0,0 @@
import { connectionMongo, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo;
import type { UserInformSchema } from '@fastgpt/global/support/user/type.d';
import { InformTypeMap } from '@fastgpt/global/support/user/constant';
const InformSchema = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
required: true
},
time: {
type: Date,
default: () => new Date()
},
type: {
type: String,
enum: Object.keys(InformTypeMap)
},
title: {
type: String,
required: true
},
content: {
type: String,
required: true
},
read: {
type: Boolean,
default: false
}
});
try {
InformSchema.index({ time: -1 });
InformSchema.index({ userId: 1 });
} catch (error) {
console.log(error);
}
export const MongoUserInform: Model<UserInformSchema> =
models['inform'] || model('inform', InformSchema);

View File

@@ -1,7 +1,7 @@
import { connectionMongo, type Model } from '../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { hashStr } from '@fastgpt/global/common/string/tools';
import { PRICE_SCALE } from '@fastgpt/global/common/bill/constants';
import { PRICE_SCALE } from '@fastgpt/global/support/wallet/bill/constants';
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
export const userCollectionName = 'users';

View File

@@ -0,0 +1,125 @@
import { TeamItemType } from '@fastgpt/global/support/user/team/type';
import { connectionMongo, Types } from '../../../common/mongo';
import {
TeamMemberRoleEnum,
TeamMemberStatusEnum,
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
export async function getTeamInfoByTmbId({
tmbId,
userId
}: {
tmbId?: string;
userId?: string;
}): Promise<TeamItemType> {
if (!tmbId && !userId) {
return Promise.reject('tmbId or userId is required');
}
const db = connectionMongo?.connection?.db;
const TeamMember = db.collection(TeamMemberCollectionName);
const results = await TeamMember.aggregate([
{
$match: tmbId
? {
_id: new Types.ObjectId(tmbId)
}
: {
userId: new Types.ObjectId(userId),
defaultTeam: true
}
},
{
$lookup: {
from: TeamCollectionName, // 关联的集合名
localField: 'teamId', // TeamMember 集合中用于关联的字段
foreignField: '_id', // Team 集合中用于关联的字段
as: 'team' // 查询结果中的字段名,存放关联查询的结果
}
},
{
$unwind: '$team' // 将查询结果中的 team 字段展开,变成一个对象
}
]).toArray();
const tmb = results[0];
if (!tmb) {
return Promise.reject('team not exist');
}
return {
userId: String(tmb.userId),
teamId: String(tmb.teamId),
teamName: tmb.team.name,
avatar: tmb.team.avatar,
balance: tmb.team.balance,
tmbId: String(tmb._id),
role: tmb.role,
status: tmb.status,
defaultTeam: tmb.defaultTeam,
canWrite: tmb.role !== TeamMemberRoleEnum.visitor,
maxSize: tmb.team.maxSize
};
}
export async function createDefaultTeam({
userId,
teamName = 'My Team',
avatar = '/icon/logo.svg',
balance = 0,
maxSize = 5
}: {
userId: string;
teamName?: string;
avatar?: string;
balance?: number;
maxSize?: number;
}) {
const db = connectionMongo.connection.db;
const Team = db.collection(TeamCollectionName);
const TeamMember = db.collection(TeamMemberCollectionName);
// auth default team
const tmb = await TeamMember.findOne({
userId: new Types.ObjectId(userId),
defaultTeam: true
});
if (!tmb) {
console.log('create default team', userId);
// create
const { insertedId } = await Team.insertOne({
ownerId: userId,
name: teamName,
avatar,
balance,
maxSize,
createTime: new Date()
});
await TeamMember.insertOne({
teamId: insertedId,
userId,
role: TeamMemberRoleEnum.owner,
status: TeamMemberStatusEnum.active,
createTime: new Date(),
defaultTeam: true
});
} else {
console.log('default team exist', userId);
await Team.updateOne(
{
_id: new Types.ObjectId(tmb.teamId)
},
{
$set: {
balance,
maxSize
}
}
);
}
}

View File

@@ -0,0 +1,60 @@
import { connectionMongo, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { BillSchema as BillType } from '@fastgpt/global/support/wallet/bill/type';
import { BillSourceMap } from '@fastgpt/global/support/wallet/bill/constants';
import {
TeamCollectionName,
TeamMemberCollectionName
} from '@fastgpt/global/support/user/team/constant';
const BillSchema = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'user'
},
teamId: {
type: Schema.Types.ObjectId,
ref: TeamCollectionName,
required: true
},
tmbId: {
type: Schema.Types.ObjectId,
ref: TeamMemberCollectionName,
required: true
},
appName: {
type: String,
default: ''
},
appId: {
type: Schema.Types.ObjectId,
ref: 'model',
required: false
},
time: {
type: Date,
default: () => new Date()
},
total: {
type: Number,
required: true
},
source: {
type: String,
enum: Object.keys(BillSourceMap),
required: true
},
list: {
type: Array,
default: []
}
});
try {
BillSchema.index({ userId: 1 });
BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 });
} catch (error) {
console.log(error);
}
export const MongoBill: Model<BillType> = models['bill'] || model('bill', BillSchema);

View File

@@ -1,31 +0,0 @@
import { connectionMongo, type Model } from '../../../common/mongo';
const { Schema, model, models } = connectionMongo;
import { PaySchema as PayType } from '@fastgpt/global/support/wallet/type.d';
const PaySchema = new Schema({
userId: {
type: Schema.Types.ObjectId,
ref: 'user',
required: true
},
createTime: {
type: Date,
default: () => new Date()
},
price: {
type: Number,
required: true
},
orderId: {
type: String,
required: true
},
status: {
// 支付的状态
type: String,
default: 'NOTPAY',
enum: ['SUCCESS', 'REFUND', 'NOTPAY', 'CLOSED']
}
});
export const MongoPay: Model<PayType> = models['pay'] || model('pay', PaySchema);