gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v4.6 -1 (#459)

Browse Source
This commit is contained in:
Archer
2023-11-09 09:46:57 +08:00
committed by GitHub
parent 661ee79943
commit 8bb5588305
402 changed files with 9899 additions and 5967 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
projects/app/src/pages/api/admin/initv44.ts
View File

@@ -1,42 +0,0 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
await MongoDataset.updateMany(
{
type: { $exists: false }
},
{
$set: {
type: DatasetTypeEnum.dataset,
parentId: null
}
}
);
const response = await PgClient.update(PgDatasetTableName, {
where: [['file_id', 'undefined']],
values: [{ key: 'file_id', value: '' }]
});
jsonRes(res, {
data: response.rowCount
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}

35
projects/app/src/pages/api/admin/initv441.ts
View File

@@ -1,35 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { connectToDatabase } from '@/service/mongo';
import mongoose from '@fastgpt/service/common/mongo';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
const data = await mongoose.connection.db
.collection('dataset.files')
.updateMany({}, { $set: { 'metadata.datasetUsed': true } });
// update pg data
const pg = await PgClient.query(`UPDATE ${PgDatasetTableName}
SET file_id = ''
WHERE (file_id = 'undefined' OR LENGTH(file_id) < 20) AND file_id != '';`);
jsonRes(res, {
data: {
data,
pg
}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}

27
projects/app/src/pages/api/admin/initv442.ts
View File

@@ -1,27 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { connectToDatabase, Bill } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
try {
await Bill.collection.dropIndex('time_1');
} catch (error) {}
try {
await Bill.collection.createIndex({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 });
} catch (error) {}
jsonRes(res, {
data: {}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}

104
projects/app/src/pages/api/admin/initv445.ts
View File

@@ -1,104 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { connectToDatabase, App } from '@/service/mongo';
import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { SystemInputEnum } from '@/constants/app';
const limit = 300;
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
const totalApps = await App.countDocuments();
// init app
await App.updateMany({}, { $set: { inited: false } });
for (let i = 0; i < totalApps; i += limit) {
await initVariable();
console.log(i + limit);
}
jsonRes(res, {
data: {
total: totalApps
}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
async function initVariable(): Promise<any> {
try {
const apps = await App.find({ inited: false }).limit(limit);
await Promise.all(
apps.map(async (app) => {
const jsonAPP = app.toObject();
// @ts-ignore
app.inited = true;
const modules = jsonAPP.modules;
// 找到 variable
const variable = modules.find((item) => item.flowType === FlowNodeTypeEnum.variable);
if (!variable) return await app.save();
// 找到 guide 模块
const userGuideModule = modules.find(
(item) => item.flowType === FlowNodeTypeEnum.userGuide
);
if (userGuideModule) {
userGuideModule.inputs = [
userGuideModule.inputs[0],
{
key: SystemInputEnum.variables,
type: FlowNodeInputTypeEnum.systemInput,
label: '对话框变量',
value: variable.inputs[0]?.value
}
];
} else {
modules.unshift({
moduleId: 'userGuide',
flowType: FlowNodeTypeEnum.userGuide,
name: '用户引导',
position: {
x: 447.98520778293346,
y: 721.4016845336229
},
inputs: [
{
key: SystemInputEnum.welcomeText,
type: FlowNodeInputTypeEnum.input,
label: '开场白'
},
{
key: SystemInputEnum.variables,
type: FlowNodeInputTypeEnum.systemInput,
label: '对话框变量',
value: variable.inputs[0]?.value
}
],
outputs: []
});
}
jsonAPP.modules = jsonAPP.modules.filter(
(item) => item.flowType !== FlowNodeTypeEnum.variable
);
app.modules = JSON.parse(JSON.stringify(jsonAPP.modules));
await app.save();
})
);
} catch (error) {
return initVariable();
}
}

93
projects/app/src/pages/api/admin/initv447.ts
View File

@@ -1,93 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { connectToDatabase } from '@/service/mongo';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { DatasetSpecialIdEnum } from '@fastgpt/global/core/dataset/constant';
import { Types, connectionMongo } from '@fastgpt/service/common/mongo';
import { delay } from '@/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
let initFileIds: string[] = [];
try {
const { limit = 100 } = req.body;
await connectToDatabase();
await authUser({ req, authRoot: true });
console.log('count rows');
// 去重获取 fileId
const { rows } = await PgClient.query(`SELECT DISTINCT file_id
FROM ${PgDatasetTableName} WHERE file_id IS NOT NULL AND file_id != '';
`);
console.log('count rows success', rows.length);
console.log('start filter');
for (let i = 0; i < rows.length; i += limit) {
await init(rows.slice(i, i + limit), initFileIds);
console.log(i);
}
for (let i = 0; i < initFileIds.length; i++) {
await PgClient.query(`UPDATE ${PgDatasetTableName}
SET file_id = '${DatasetSpecialIdEnum.manual}'
WHERE file_id = '${initFileIds[i]}'`);
console.log('update: ', initFileIds[i]);
}
const { rows: emptyIds } = await PgClient.query(
`SELECT id FROM ${PgDatasetTableName} WHERE file_id IS NULL OR file_id=''`
);
console.log('filter success');
console.log(emptyIds.length);
await delay(5000);
console.log('start update');
async function start(start: number) {
for (let i = start; i < emptyIds.length; i += limit) {
await PgClient.query(`UPDATE ${PgDatasetTableName}
SET file_id = '${DatasetSpecialIdEnum.manual}'
WHERE id = '${emptyIds[i].id}'`);
console.log('update: ', i, emptyIds[i].id);
}
}
for (let i = 0; i < limit; i++) {
start(i);
}
console.log('update success');
jsonRes(res, {
data: {
empty: emptyIds.length
}
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
async function init(rows: any[], initFileIds: string[]) {
const collection = connectionMongo.connection.db.collection(`dataset.files`);
/* 遍历所有的 fileId，去找有没有对应的文件，没有的话则改成manual */
const updateResult = await Promise.allSettled(
rows.map(async (item) => {
// 找下是否有对应的文件
const file = await collection.findOne({
_id: new Types.ObjectId(item.file_id)
});
if (file) return '';
// 没有文件的，改成manual
initFileIds.push(item.file_id);
return item.file_id;
})
);
// @ts-ignore
console.log(updateResult.filter((item) => item?.value).length);
}

344
projects/app/src/pages/api/admin/initv451.ts
View File

@@ -1,344 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { App, connectToDatabase } from '@/service/mongo';
import { PgClient } from '@/service/pg';
import { connectionMongo } from '@fastgpt/service/common/mongo';
import { PgDatasetTableName } from '@/constants/plugin';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { delay } from '@/utils/tools';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { strIsLink } from '@fastgpt/global/common/string/tools';
import { GridFSStorage } from '@/service/lib/gridfs';
import { Types } from 'mongoose';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await connectToDatabase();
console.log('rename');
await rename();
console.log('init mongo data');
await initMongo(limit);
console.log('create collection');
await createCollection();
console.log('update pg collectionId');
await updatePgCollection();
console.log('init done');
jsonRes(res, {
data: {}
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
async function rename() {
// rename mongo kbs -> datasets
try {
const collections = await connectionMongo.connection.db
.listCollections({ name: 'kbs' })
.toArray();
if (collections.length > 0) {
const kbCollection = connectionMongo.connection.db.collection('kbs');
await kbCollection.rename('datasets', { dropTarget: true });
console.log('success rename kbs -> datasets');
}
} catch (error) {
console.log('error： rename kbs -> datasets', error);
}
// rename pg: kb_id -> dataset_id
try {
const { rows } = await PgClient.query(`SELECT EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_name = '${PgDatasetTableName}'
AND column_name = 'kb_id'
);`);
if (rows[0].exists) {
await PgClient.query(`ALTER TABLE ${PgDatasetTableName} RENAME COLUMN kb_id TO dataset_id`);
console.log('success rename kb_id -> dataset_id');
}
} catch (error) {
console.log('error： rename kb_id -> dataset_id', error);
}
// rename pg: file_id -> collection_id
try {
const { rows } = await PgClient.query(`SELECT EXISTS (
SELECT 1
FROM information_schema.columns
WHERE table_name = '${PgDatasetTableName}'
AND column_name = 'file_id'
);`);
if (rows[0].exists) {
await PgClient.query(
`ALTER TABLE ${PgDatasetTableName} RENAME COLUMN file_id TO collection_id`
);
console.log('success rename file_id -> collection_id');
}
} catch (error) {
console.log('error： rename file_id -> collection_id', error);
}
}
async function initMongo(limit: number) {
let success = 0;
async function initApp(limit = 100): Promise<any> {
// 遍历所有 app，更新 app modules 里的 FlowNodeTypeEnum.kbSearchNode
const apps = await App.find({ inited: false }).limit(limit);
if (apps.length === 0) return;
try {
await Promise.all(
apps.map(async (app) => {
const modules = app.toObject().modules;
// @ts-ignore
app.inited = true;
modules.forEach((module) => {
// @ts-ignore
if (module.flowType === 'kbSearchNode') {
module.flowType = FlowNodeTypeEnum.datasetSearchNode;
module.inputs.forEach((input) => {
if (input.key === 'kbList') {
input.key = 'datasets';
input.value?.forEach((item: any) => {
item.datasetId = item.kbId;
});
}
});
}
});
app.modules = JSON.parse(JSON.stringify(modules));
await app.save();
})
);
success += limit;
console.log('mongo init:', success);
return initApp(limit);
} catch (error) {
return initApp(limit);
}
}
// init app
await App.updateMany(
{},
{
$set: {
inited: false
}
}
);
const totalApp = await App.countDocuments();
console.log(`total app: ${totalApp}`);
await delay(2000);
console.log('start init app');
await initApp(limit);
console.log('init mongo success');
}
type RowType = { user_id: string; dataset_id: string; collection_id: string };
async function createCollection() {
let success = 0;
const { rows, rowCount } = await PgClient.query(`SELECT user_id,dataset_id,collection_id
FROM ${PgDatasetTableName}
GROUP BY user_id,collection_id, dataset_id
ORDER BY dataset_id`);
if (rowCount === 0) {
console.log('pg done');
return;
}
// init dataset collection
console.log(`total collection: ${rowCount}`);
// collectionId 的类型：manual, mark, httpLink, fileId
async function initCollection(row: RowType): Promise<any> {
try {
{
const userId = row.user_id;
const datasetId = row.dataset_id;
const collectionId = row.collection_id;
const count = await MongoDatasetCollection.countDocuments({
datasetId,
userId,
['metadata.pgCollectionId']: collectionId
});
if (count > 0) {
console.log('collection already exist');
return;
}
if (collectionId === 'manual') {
await MongoDatasetCollection.create({
parentId: null,
datasetId,
userId,
name: '手动录入',
type: DatasetCollectionTypeEnum.virtual,
updateTime: new Date('2099'),
metadata: {
pgCollectionId: collectionId
}
});
} else if (collectionId === 'mark') {
await MongoDatasetCollection.create({
parentId: null,
datasetId,
userId,
name: '手动标注',
type: DatasetCollectionTypeEnum.virtual,
updateTime: new Date('2099'),
metadata: {
pgCollectionId: collectionId
}
});
} else if (strIsLink(collectionId)) {
await MongoDatasetCollection.create({
parentId: null,
datasetId,
userId,
name: collectionId,
type: DatasetCollectionTypeEnum.link,
metadata: {
rawLink: collectionId,
pgCollectionId: collectionId
}
});
} else {
// find file
const gridFs = new GridFSStorage('dataset', userId);
const collection = gridFs.Collection();
const file = await collection.findOne({
_id: new Types.ObjectId(collectionId)
});
if (file) {
await MongoDatasetCollection.create({
parentId: null,
datasetId,
userId,
name: file.filename,
type: DatasetCollectionTypeEnum.file,
metadata: {
fileId: file._id,
pgCollectionId: collectionId
}
});
} else {
// no file
await MongoDatasetCollection.create({
parentId: null,
datasetId,
userId,
name: '未知文件',
type: DatasetCollectionTypeEnum.virtual,
metadata: {
pgCollectionId: collectionId
}
});
}
}
console.log('create collection success');
}
} catch (error) {
console.log(error);
await delay(2000);
return initCollection(row);
}
}
for await (const row of rows) {
await initCollection(row);
console.log('init collection success: ', ++success);
}
}
async function updatePgCollection(): Promise<any> {
let success = 0;
const limit = 10;
const collections = await MongoDatasetCollection.find({
'metadata.pgCollectionId': { $exists: true, $ne: '' }
}).lean();
console.log('total:', collections.length);
async function update(i: number): Promise<any> {
const item = collections[i];
if (!item) return;
try {
console.log('start', item.name, item.datasetId, item.metadata.pgCollectionId);
const time = Date.now();
if (item.metadata.pgCollectionId) {
const { rows } = await PgClient.select(PgDatasetTableName, {
fields: ['id'],
where: [
['dataset_id', String(item.datasetId)],
'AND',
['collection_id', String(item.metadata.pgCollectionId)]
],
limit: 999999
});
console.log('update date total', rows.length, 'time:', Date.now() - time);
await PgClient.query(`
update ${PgDatasetTableName} set collection_id = '${item._id}' where dataset_id = '${String(
item.datasetId
)}' AND collection_id = '${String(item.metadata.pgCollectionId)}'
`);
console.log('pg update time', Date.now() - time);
}
// 更新 file id
if (item.type === 'file' && item.metadata.fileId) {
const collection = connectionMongo.connection.db.collection(`dataset.files`);
await collection.findOneAndUpdate({ _id: new Types.ObjectId(item.metadata.fileId) }, [
{
$set: {
'metadata.datasetId': item.datasetId,
'metadata.collectionId': item._id
}
}
]);
}
await MongoDatasetCollection.findByIdAndUpdate(item._id, {
$unset: { 'metadata.pgCollectionId': '' }
});
console.log('success', ++success);
return update(i + limit);
} catch (error) {
console.log(error);
await delay(5000);
return update(i);
}
}
const arr = new Array(limit).fill(0);
return Promise.all(arr.map((_, i) => update(i)));
}

335
projects/app/src/pages/api/admin/initv46.ts Normal file
View File

@@ -0,0 +1,335 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema';
import {
createDefaultTeam,
getTeamInfoByTmbId
} from '@fastgpt/service/support/user/team/controller';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { UserModelSchema } from '@fastgpt/global/support/user/type';
import { delay } from '@/utils/tools';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import {
DatasetCollectionSchemaType,
DatasetSchemaType,
DatasetTrainingSchemaType
} from '@fastgpt/global/core/dataset/type';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { connectionMongo } from '@fastgpt/service/common/mongo';
import { Types } from 'mongoose';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { PgClient } from '@fastgpt/service/common/pg';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
import { POST } from '@fastgpt/service/common/api/plusRequest';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50, maxSize = 3 } = req.body as { limit: number; maxSize: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
await initDefaultTeam(limit, maxSize);
await initMongoTeamId(limit);
await initDatasetAndApp();
await initCollectionFileTeam(limit);
if (global.systemEnv.pluginBaseUrl) {
POST('/admin/init46');
}
await initPgData();
jsonRes(res, {
data: {}
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
async function initDefaultTeam(limit: number, maxSize: number) {
/* init user default Team */
const users = await MongoUser.find({}, '_id balance');
console.log('init user default team', users.length);
// 100 组一次
const userArr: UserModelSchema[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const users of userArr) {
await Promise.all(users.map(init));
success += limit;
console.log(success);
}
async function init(user: UserModelSchema): Promise<any> {
try {
await createDefaultTeam({
userId: user._id,
balance: user.balance,
maxSize
});
} catch (error) {
console.log(error);
await delay(1000);
return init(user);
}
}
}
async function initMongoTeamId(limit: number) {
const mongoSchema = [
{
label: 'MongoPlugin',
schema: MongoPlugin
},
{
label: 'MongoChat',
schema: MongoChat
},
{
label: 'MongoChatItem',
schema: MongoChatItem
},
{
label: 'MongoApp',
schema: MongoApp
},
{
label: 'MongoDataset',
schema: MongoDataset
},
{
label: 'MongoDatasetCollection',
schema: MongoDatasetCollection
},
{
label: 'MongoDatasetTraining',
schema: MongoDatasetTraining
},
{
label: 'MongoBill',
schema: MongoBill
},
{
label: 'MongoOutLink',
schema: MongoOutLink
},
{
label: 'MongoOpenApi',
schema: MongoOpenApi
}
];
/* init user default Team */
for await (const item of mongoSchema) {
console.log('start init', item.label);
await initTeamTmbId(item.schema);
console.log('finish init', item.label);
}
async function initTeamTmbId(schema: any) {
const emptyWhere = {
$or: [{ teamId: { $exists: false } }, { teamId: null }]
};
const uniqueUsersWithNoTeamId = await schema.aggregate([
{
$match: emptyWhere
},
{
$group: {
_id: '$userId', // 按 userId 分组以去重
userId: { $first: '$userId' } // 保留第一个出现的 userId
}
},
{
$project: {
_id: 0, // 不显示 _id 字段
userId: 1 // 只显示 userId 字段
}
}
]);
const users = uniqueUsersWithNoTeamId;
console.log('un init total', users.length);
// limit 组一次
const userArr: any[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const users of userArr) {
await Promise.all(users.map((item) => init(item.userId)));
success += limit;
console.log(success);
}
async function init(userId: string): Promise<any> {
try {
const tmb = await getTeamInfoByTmbId({ userId });
await schema.updateMany(
{
userId,
...emptyWhere
},
{
teamId: tmb.teamId,
tmbId: tmb.tmbId
}
);
} catch (error) {
if (error === 'team not exist' || error === 'tmbId or userId is required') {
return;
}
console.log(error);
await delay(1000);
return init(userId);
}
}
}
}
async function initDatasetAndApp() {
await MongoDataset.updateMany(
{},
{
$set: {
permission: PermissionTypeEnum.private
}
}
);
await MongoApp.updateMany(
{},
{
$set: {
permission: PermissionTypeEnum.private
}
}
);
}
async function initCollectionFileTeam(limit: number) {
/* init user default Team */
const DatasetFile = connectionMongo.connection.db.collection(`dataset.files`);
const matchWhere = {
$or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }]
};
const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([
{
$match: matchWhere
},
{
$group: {
_id: '$metadata.userId', // 按 metadata.userId 分组以去重
userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId
}
},
{
$project: {
_id: 0, // 不显示 _id 字段
userId: 1 // 只显示 userId 字段
}
}
]).toArray();
const users = uniqueUsersWithNoTeamId;
console.log('un init total', users.length);
// limit 组一次
const userArr: any[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const item of userArr) {
await Promise.all(item.map((item) => init(item.userId)));
success += limit;
console.log(success);
}
async function init(userId: string): Promise<any> {
try {
const tmb = await getTeamInfoByTmbId({
userId
});
await DatasetFile.updateMany(
{
userId,
...matchWhere
},
{
$set: {
'metadata.teamId': String(tmb.teamId),
'metadata.tmbId': String(tmb.tmbId)
}
}
);
} catch (error) {
if (error === 'team not exist' || error === 'tmbId or userId is required') {
return;
}
console.log(error);
await delay(1000);
return init(userId);
}
}
}
async function initPgData() {
const limit = 10;
// add column
try {
await Promise.all([
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id CHAR(50);`),
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id CHAR(50);`),
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN user_id DROP NOT NULL;`)
]);
} catch (error) {
console.log(error);
console.log('column exits');
}
const { rows } = await PgClient.query<{ user_id: string }>(`
SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id IS NULL;
`);
console.log('init pg', rows.length);
let success = 0;
for (let i = 0; i < limit; i++) {
init(i);
}
async function init(index: number): Promise<any> {
const userId = rows[index]?.user_id;
if (!userId) return;
try {
const tmb = await getTeamInfoByTmbId({ userId });
// update pg
await PgClient.query(
`Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id IS NULL;`
);
console.log(++success);
init(index + limit);
} catch (error) {
if (error === 'default team not exist') {
return;
}
console.log(error);
await delay(1000);
return init(index);
}
}
}

32
projects/app/src/pages/api/app/myApps.ts
View File

@@ -1,32 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, App } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { AppListItemType } from '@/types/app';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
// 根据 userId 获取模型信息
const myApps = await App.find(
{
userId
},
'_id avatar name intro'
).sort({
updateTime: -1
});
jsonRes<AppListItemType[]>(res, {
data: myApps
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

37
projects/app/src/pages/api/chat/history/updateChatHistory.ts
View File

@@ -1,37 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Chat } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
export type Props = {
chatId: string;
customTitle?: string;
top?: boolean;
};
/* 更新聊天标题 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { chatId, customTitle, top } = req.body as Props;
const { userId } = await authUser({ req, authToken: true });
await Chat.findOneAndUpdate(
{
chatId,
userId
},
{
...(customTitle ? { customTitle } : {}),
...(top ? { top } : { top: null })
}
);
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

110
projects/app/src/pages/api/chat/init.ts
View File

@@ -1,110 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { Chat, ChatItem, connectToDatabase } from '@/service/mongo';
import type { InitChatResponse } from '@/global/core/api/chatRes.d';
import { authUser } from '@fastgpt/service/support/user/auth';
import { ChatItemType } from '@/types/chat';
import { authApp } from '@/service/utils/auth';
import type { ChatSchema } from '@/types/mongoSchema';
import { getGuideModule } from '@/global/core/app/modules/utils';
import { getChatModelNameListByModules } from '@/service/core/app/module';
import { TaskResponseKeyEnum } from '@/constants/chat';
/* 初始化我的聊天框，需要身份验证 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
let { appId, chatId } = req.query as {
appId: '' | string;
chatId: '' | string;
};
if (!appId) {
return jsonRes(res, {
code: 501,
message: "You don't have an app yet"
});
}
// 校验使用权限
const app = (
await authApp({
appId,
userId,
authUser: false,
authOwner: false
})
).app;
// get app and history
const { chat, history = [] }: { chat?: ChatSchema; history?: ChatItemType[] } =
await (async () => {
if (chatId) {
// auth chatId
const [chat, history] = await Promise.all([
Chat.findOne(
{
chatId,
userId,
appId
},
'title variables'
),
ChatItem.find(
{
chatId,
userId,
appId
},
`dataId obj value adminFeedback userFeedback ${TaskResponseKeyEnum.responseData}`
)
.sort({ _id: -1 })
.limit(30)
]);
if (!chat) {
throw new Error('聊天框不存在');
}
history.reverse();
return { app, history, chat };
}
return {};
})();
if (!app) {
throw new Error('Auth App Error');
}
const isOwner = String(app.userId) === userId;
jsonRes<InitChatResponse>(res, {
data: {
chatId,
appId,
app: {
userGuideModule: getGuideModule(app.modules),
chatModels: getChatModelNameListByModules(app.modules),
name: app.name,
avatar: app.avatar,
intro: app.intro,
canUse: app.share.isShare || isOwner
},
title: chat?.title || '新对话',
variables: chat?.variables || {},
history
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
export const config = {
api: {
responseLimit: '10mb'
}
};

56
projects/app/src/pages/api/chat/removeHistory.ts
View File

@@ -1,56 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Chat, ChatItem } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { ChatSourceEnum } from '@/constants/chat';
type Props = {
chatId?: string;
appId?: string;
};
/* clear chat history */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { chatId, appId } = req.query as Props;
const { userId } = await authUser({ req, authToken: true });
if (chatId) {
await Promise.all([
Chat.findOneAndRemove({
chatId,
userId
}),
ChatItem.deleteMany({
userId,
chatId
})
]);
}
if (appId) {
const chats = await Chat.find({
appId,
userId,
source: ChatSourceEnum.online
}).select('_id');
const chatIds = chats.map((chat) => chat._id);
await Promise.all([
Chat.deleteMany({
_id: { $in: chatIds }
}),
ChatItem.deleteMany({
chatId: { $in: chatIds }
})
]);
}
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

14
projects/app/src/pages/api/system/file/read.ts → projects/app/src/pages/api/common/file/read.ts
View File

@@ -1,9 +1,9 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { GridFSStorage } from '@/service/lib/gridfs';
import { authFileToken } from './readUrl';
import { authFileToken } from '@fastgpt/service/support/permission/controller';
import jschardet from 'jschardet';
import { getDownloadBuf, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -11,17 +11,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { token } = req.query as { token: string };
const { fileId, userId } = await authFileToken(token);
const { fileId, teamId, tmbId, bucketName } = await authFileToken(token);
if (!fileId) {
throw new Error('fileId is empty');
}
const gridFs = new GridFSStorage('dataset', userId);
const [file, buffer] = await Promise.all([
gridFs.findAndAuthFile(fileId),
gridFs.download(fileId)
getFileById({ bucketName, fileId }),
getDownloadBuf({ bucketName, fileId })
]);
const encoding = jschardet.detect(buffer)?.encoding;

25
projects/app/src/pages/api/system/file/upload.ts → projects/app/src/pages/api/common/file/upload.ts
View File

@@ -1,11 +1,12 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { customAlphabet } from 'nanoid';
import multer from 'multer';
import path from 'path';
import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
const nanoid = customAlphabet('1234567890abcdef', 12);
@@ -38,7 +39,11 @@ class UploadModel {
}).any();
async doUpload(req: NextApiRequest, res: NextApiResponse) {
return new Promise<{ files: FileType[]; metadata: Record<string, any> }>((resolve, reject) => {
return new Promise<{
files: FileType[];
bucketName: `${BucketNameEnum}`;
metadata: Record<string, any>;
}>((resolve, reject) => {
// @ts-ignore
this.uploader(req, res, (error) => {
if (error) {
@@ -52,6 +57,7 @@ class UploadModel {
...file,
originalname: decodeURIComponent(file.originalname)
})) || [],
bucketName: req.body.bucketName,
metadata: (() => {
if (!req.body?.metadata) return {};
try {
@@ -73,15 +79,16 @@ const upload = new UploadModel();
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { userId, teamId, tmbId } = await authCert({ req, authToken: true });
const { files, metadata } = await upload.doUpload(req, res);
const gridFs = new GridFSStorage('dataset', userId);
const { files, bucketName, metadata } = await upload.doUpload(req, res);
const upLoadResults = await Promise.all(
files.map((file) =>
gridFs.save({
uploadFile({
teamId,
tmbId,
bucketName,
path: file.path,
filename: file.originalname,
metadata: {

6
projects/app/src/pages/api/system/file/uploadImage.ts → projects/app/src/pages/api/common/file/uploadImage.ts
View File

@@ -1,7 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller';
type Props = { base64Img: string };
@@ -9,7 +9,7 @@ type Props = { base64Img: string };
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { userId } = await authCert({ req, authToken: true });
const { base64Img } = req.body as Props;
const data = await uploadMongoImg({

6
projects/app/src/pages/api/user/account/paySuccess.ts → projects/app/src/pages/api/common/system/unlockTask.ts
View File

@@ -1,14 +1,14 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { startQueue } from '@/service/utils/tools';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { userId } = await authCert({ req, authToken: true });
await unlockTask(userId);
} catch (error) {}
jsonRes(res);

22
projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts
View File

@@ -1,27 +1,20 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import type { CreateQuestionGuideParams } from '@/global/core/api/aiReq.d';
import { pushQuestionGuideBill } from '@/service/common/bill/push';
import type { CreateQuestionGuideParams } from '@/global/core/ai/api.d';
import { pushQuestionGuideBill } from '@/service/support/wallet/bill/push';
import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { messages } = req.body as CreateQuestionGuideParams;
const { user } = await authUser({
const { tmbId, teamId } = await authCert({
req,
authOutLink: true,
authToken: true,
authApiKey: true,
authBalance: true
authToken: true
});
if (!user) {
throw new Error('user not found');
}
const qgModel = global.qgModels[0];
const { result, tokens } = await createQuestionGuide({
@@ -35,7 +28,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
pushQuestionGuideBill({
tokens: tokens,
userId: user._id
teamId,
tmbId
});
} catch (err) {
jsonRes(res, {

23
projects/app/src/pages/api/app/create.ts → projects/app/src/pages/api/core/app/create.ts
View File

@@ -1,11 +1,11 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { App } from '@/service/models/app';
import type { CreateAppParams } from '@/types/app';
import { AppTypeEnum } from '@/constants/app';
import type { CreateAppParams } from '@fastgpt/global/core/app/api.d';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -22,21 +22,22 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true });
// 上限校验
const authCount = await App.countDocuments({
userId
const authCount = await MongoApp.countDocuments({
teamId
});
if (authCount >= 50) {
throw new Error('上限 50 个应用');
throw new Error('每个团队上限 50 个应用');
}
// 创建模型
const response = await App.create({
const response = await MongoApp.create({
avatar,
name,
userId,
teamId,
tmbId,
modules,
type
});

11
projects/app/src/pages/api/app/data/totalUsage.ts → projects/app/src/pages/api/core/app/data/totalUsage.ts
View File

@@ -1,16 +1,17 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Bill } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { Types } from '@fastgpt/service/common/mongo';
import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId, start, end } = req.body as { appId: string; start: number; end: number };
const { userId } = await authUser({ req, authToken: true });
const { userId } = await authCert({ req, authToken: true });
const result = await Bill.aggregate([
const result = await MongoBill.aggregate([
{
$match: {
appId: new Types.ObjectId(appId),

24
projects/app/src/pages/api/app/del.ts → projects/app/src/pages/api/core/app/del.ts
View File

@@ -1,9 +1,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { Chat, App, connectToDatabase } from '@/service/mongo';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authApp } from '@/service/utils/auth';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -16,16 +17,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
// 验证是否是该用户的 app
await authApp({
appId,
userId
});
await authApp({ req, authToken: true, appId, per: 'owner' });
// 删除对应的聊天
await Chat.deleteMany({
await MongoChat.deleteMany({
appId
});
@@ -35,9 +30,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// 删除模型
await App.deleteOne({
_id: appId,
userId
await MongoApp.deleteOne({
_id: appId
});
jsonRes(res);

12
projects/app/src/pages/api/app/detail.tsx → projects/app/src/pages/api/core/app/detail.tsx
View File

@@ -1,8 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authApp } from '@/service/utils/auth';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -15,12 +14,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { app } = await authApp({
appId,
userId
});
const { app } = await authApp({ req, authToken: true, appId, per: 'w' });
jsonRes(res, {
data: app

16
projects/app/src/pages/api/app/getChatLogs.ts → projects/app/src/pages/api/core/app/getChatLogs.ts
View File

@@ -1,12 +1,13 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { Chat, connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import type { PagingData } from '@/types';
import { AppLogsListItemType } from '@/types/app';
import { Types } from '@fastgpt/service/common/mongo';
import { addDays } from 'date-fns';
import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -24,11 +25,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { teamId } = await authApp({ req, authToken: true, appId, per: 'w' });
const where = {
appId: new Types.ObjectId(appId),
userId: new Types.ObjectId(userId),
teamId: new Types.ObjectId(teamId),
updateTime: {
$gte: new Date(dateStart),
$lte: new Date(dateEnd)
@@ -36,7 +37,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
};
const [data, total] = await Promise.all([
Chat.aggregate([
MongoChat.aggregate([
{ $match: where },
{
$lookup: {
@@ -84,6 +85,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
{ $limit: pageSize },
{
$project: {
_id: 1,
id: '$chatId',
title: 1,
source: 1,
@@ -94,7 +96,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
}
]),
Chat.countDocuments(where)
MongoChat.countDocuments(where)
]);
jsonRes<PagingData<AppLogsListItemType>>(res, {

38
projects/app/src/pages/api/core/app/list.ts Normal file
View File

@@ -0,0 +1,38 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { AppListItemType } from '@fastgpt/global/core/app/type';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
// 凭证校验
const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true });
// 根据 userId 获取模型信息
const myApps = await MongoApp.find(
{ ...mongoRPermission({ teamId, tmbId, role }) },
'_id avatar name intro tmbId permission'
).sort({
updateTime: -1
});
jsonRes<AppListItemType[]>(res, {
data: myApps.map((app) => ({
_id: app._id,
avatar: app.avatar,
name: app.name,
intro: app.intro,
isOwner: teamOwner || String(app.tmbId) === tmbId,
permission: app.permission
}))
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

28
projects/app/src/pages/api/app/update.ts → projects/app/src/pages/api/core/app/update.ts
View File

@@ -1,17 +1,16 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { App } from '@/service/models/app';
import type { AppUpdateParams } from '@/types/app';
import { authApp } from '@/service/utils/auth';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import type { AppUpdateParams } from '@fastgpt/global/core/app/api';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { SystemOutputEnum } from '@/constants/app';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { name, avatar, type, share, intro, modules } = req.body as AppUpdateParams;
const { name, avatar, type, intro, modules, permission } = req.body as AppUpdateParams;
const { appId } = req.query as { appId: string };
if (!appId) {
@@ -19,28 +18,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await authApp({
appId,
userId
});
await authApp({ req, authToken: true, appId, per: permission ? 'owner' : 'w' });
// 更新模型
await App.updateOne(
await MongoApp.updateOne(
{
_id: appId,
userId
_id: appId
},
{
name,
type,
avatar,
intro,
...(share && {
'share.isShare': share.isShare,
'share.isShareDetail': share.isShareDetail
}),
permission,
...(modules && {
modules: modules.map((modules) => ({
...modules,

33
projects/app/src/pages/api/chat/chatTest.ts → projects/app/src/pages/api/core/chat/chatTest.ts
View File

@@ -1,14 +1,15 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { sseErrRes } from '@/service/response';
import { sseResponseEventEnum } from '@/constants/chat';
import { sseErrRes } from '@fastgpt/service/common/response';
import { sseResponseEventEnum } from '@fastgpt/service/common/response/constant';
import { responseWrite } from '@fastgpt/service/common/response';
import type { ModuleItemType } from '@fastgpt/global/core/module/type.d';
import { dispatchModules } from '@/pages/api/v1/chat/completions';
import { pushChatBill } from '@/service/common/bill/push';
import { BillSourceEnum } from '@/constants/user';
import { ChatItemType } from '@/types/chat';
import { pushChatBill } from '@/service/support/wallet/bill/push';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import type { ChatItemType } from '@fastgpt/global/core/chat/type';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { authUser } from '@/service/support/permission/auth/user';
import { dispatchModules } from '@/service/moduleDispatch';
export type Props = {
history: ChatItemType[];
@@ -39,17 +40,22 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
/* user auth */
const { userId, user } = await authUser({ req, authToken: true, authBalance: true });
if (!user) {
throw new Error('user not found');
}
const [{ teamId, tmbId }, { user }] = await Promise.all([
authApp({ req, authToken: true, appId, per: 'r' }),
authUser({
req,
authToken: true,
minBalance: 0
})
]);
/* start process */
const { responseData } = await dispatchModules({
res,
modules: modules,
variables,
teamId,
tmbId,
user,
params: {
history,
@@ -74,7 +80,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
pushChatBill({
appName,
appId,
userId,
teamId,
tmbId,
source: BillSourceEnum.fastgpt,
response: responseData
});

54
projects/app/src/pages/api/core/chat/delete.ts Normal file
View File

@@ -0,0 +1,54 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
type Props = {
chatId?: string;
appId?: string;
};
/* clear chat history */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { chatId, appId } = req.query as Props;
const { tmbId } = await authCert({ req, authToken: true });
if (chatId) {
await MongoChatItem.deleteMany({
chatId,
tmbId
});
await MongoChat.findOneAndRemove({
chatId,
tmbId
});
}
if (appId) {
const chats = await MongoChat.find({
appId,
tmbId,
source: ChatSourceEnum.online
}).select('_id');
const chatIds = chats.map((chat) => chat._id);
await MongoChatItem.deleteMany({
chatId: { $in: chatIds }
});
await MongoChat.deleteMany({
_id: { $in: chatIds }
});
}
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

13
projects/app/src/pages/api/chat/feedback/adminUpdate.ts → projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts
View File

@@ -1,8 +1,9 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, ChatItem } from '@/service/mongo';
import type { AdminUpdateFeedbackParams } from '@/global/core/api/chatReq.d';
import { authUser } from '@fastgpt/service/support/user/auth';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import type { AdminUpdateFeedbackParams } from '@fastgpt/global/core/chat/api.d';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
/* 初始化我的聊天框，需要身份验证 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -14,9 +15,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('missing parameter');
}
const { userId } = await authUser({ req, authToken: true });
const { userId } = await authCert({ req, authToken: true });
await ChatItem.findOneAndUpdate(
await MongoChatItem.findOneAndUpdate(
{
userId,
dataId: chatItemId

7
projects/app/src/pages/api/chat/feedback/userUpdate.ts → projects/app/src/pages/api/core/chat/feedback/userUpdate.ts
View File

@@ -1,6 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, ChatItem } from '@/service/mongo';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
/* 初始化我的聊天框，需要身份验证 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -15,7 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('chatItemId is required');
}
await ChatItem.findOneAndUpdate(
await MongoChatItem.findOneAndUpdate(
{
dataId: chatItemId
},

97
projects/app/src/pages/api/core/chat/init.ts Normal file
View File

@@ -0,0 +1,97 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import type { InitChatResponse } from '@fastgpt/global/core/chat/api.d';
import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import type { ChatSchema } from '@fastgpt/global/core/chat/type.d';
import { getGuideModule } from '@/global/core/app/modules/utils';
import { getChatModelNameListByModules } from '@/service/core/app/module';
import { TaskResponseKeyEnum } from '@fastgpt/global/core/chat/constants';
import { authChat } from '@fastgpt/service/support/permission/auth/chat';
/* 初始化我的聊天框，需要身份验证 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
let { appId, chatId } = req.query as {
appId: string;
chatId: '' | string;
};
if (!appId) {
return jsonRes(res, {
code: 501,
message: "You don't have an app yet"
});
}
// 校验使用权限
const [{ app }, autChatResult] = await Promise.all([
authApp({
req,
authToken: true,
appId,
per: 'r'
}),
chatId
? authChat({
req,
authToken: true,
chatId,
per: 'r'
})
: undefined
]);
// get app and history
const { history = [] }: { history?: ChatItemType[] } = await (async () => {
if (chatId) {
// auth chatId
const history = await MongoChatItem.find(
{
chatId
},
`dataId obj value adminFeedback userFeedback ${TaskResponseKeyEnum.responseData}`
)
.sort({ _id: -1 })
.limit(30);
history.reverse();
return { history };
}
return {};
})();
jsonRes<InitChatResponse>(res, {
data: {
chatId,
appId,
app: {
userGuideModule: getGuideModule(app.modules),
chatModels: getChatModelNameListByModules(app.modules),
name: app.name,
avatar: app.avatar,
intro: app.intro
},
title: autChatResult?.chat?.title || '新对话',
variables: autChatResult?.chat?.variables || {},
history
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
export const config = {
api: {
responseLimit: '10mb'
}
};

16
projects/app/src/pages/api/chat/delChatRecordByContentId.ts → projects/app/src/pages/api/core/chat/item/delete.ts
View File

@@ -1,21 +1,19 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, ChatItem } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { authChat } from '@fastgpt/service/support/permission/auth/chat';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { chatId, contentId } = req.query as { chatId: string; contentId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await authChat({ req, authToken: true, chatId, per: 'w' });
// 删除一条数据库记录
await ChatItem.deleteOne({
await MongoChatItem.deleteOne({
dataId: contentId,
chatId,
userId
chatId
});
jsonRes(res);

72
projects/app/src/pages/api/core/chat/item/getSpeech.ts Normal file
View File

@@ -0,0 +1,72 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { GetChatSpeechProps } from '@/global/core/chat/api.d';
import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
import { pushAudioSpeechBill } from '@/service/support/wallet/bill/push';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { authType2BillSource } from '@/service/support/wallet/bill/utils';
/*
1. get tts from chatItem store
2. get tts from ai
3. save tts to chatItem store if chatItemId is provided
4. push bill
*/
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { chatItemId, ttsConfig, input } = req.body as GetChatSpeechProps;
const { teamId, tmbId, authType } = await authCert({ req, authToken: true });
const chatItem = await (async () => {
if (!chatItemId) return null;
return await MongoChatItem.findOne(
{
dataId: chatItemId
},
'tts'
);
})();
if (chatItem?.tts) {
return jsonRes(res, {
data: chatItem.tts
});
}
const { tts, model } = await text2Speech({
model: ttsConfig.model,
voice: ttsConfig.voice,
input
});
(async () => {
if (!chatItem) return;
try {
chatItem.tts = tts;
await chatItem.save();
} catch (error) {}
})();
jsonRes(res, {
data: tts
});
pushAudioSpeechBill({
model: model,
textLength: input.length,
tmbId,
teamId,
source: authType2BillSource({ authType })
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

23
projects/app/src/pages/api/chat/history/getHistory.ts → projects/app/src/pages/api/core/chat/list.ts
View File

@@ -1,22 +1,23 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Chat } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import type { ChatHistoryItemType } from '@/types/chat';
import { ChatSourceEnum } from '@/constants/chat';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import type { ChatHistoryItemType } from '@fastgpt/global/core/chat/type.d';
import { ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
/* 获取历史记录 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId } = req.body as { appId?: string };
const { userId } = await authUser({ req, authToken: true });
const { appId } = req.body as { appId: string };
const { tmbId } = await authApp({ req, authToken: true, appId, per: 'r' });
const data = await Chat.find(
const data = await MongoChat.find(
{
userId,
source: ChatSourceEnum.online,
...(appId && { appId })
appId,
tmbId,
source: ChatSourceEnum.online
},
'chatId title top customTitle appId updateTime'
)

27
projects/app/src/pages/api/core/chat/update.ts Normal file
View File

@@ -0,0 +1,27 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { UpdateHistoryProps } from '@fastgpt/global/core/chat/api.d';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { authChat } from '@fastgpt/service/support/permission/auth/chat';
/* 更新聊天标题 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { chatId, customTitle, top } = req.body as UpdateHistoryProps;
await authChat({ req, authToken: true, chatId });
await MongoChat.findByIdAndUpdate(chatId, {
...(customTitle ? { customTitle } : {}),
...(top ? { top } : { top: null })
});
jsonRes(res);
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

18
projects/app/src/pages/api/core/dataset/allDataset.ts
View File

@@ -1,28 +1,32 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { getVectorModel } from '@/service/core/ai/model';
import type { DatasetsItemType } from '@/types/core/dataset';
import type { DatasetItemType } from '@/types/core/dataset';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { teamId, tmbId, teamOwner, role } = await authUserRole({ req, authToken: true });
const datasets = await MongoDataset.find({
userId,
...mongoRPermission({ teamId, tmbId, role }),
type: 'dataset'
});
const data = datasets.map((item) => ({
...item.toJSON(),
vectorModel: getVectorModel(item.vectorModel)
tags: item.tags.join(' '),
vectorModel: getVectorModel(item.vectorModel),
canWrite: String(item.tmbId) === tmbId,
isOwner: teamOwner || String(item.tmbId) === tmbId
}));
jsonRes<DatasetsItemType[]>(res, {
jsonRes<DatasetItemType[]>(res, {
data
});
} catch (err) {

43
projects/app/src/pages/api/core/dataset/collection/create.ts
View File

@@ -1,28 +1,35 @@
/*
Create one dataset collection
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import type { CreateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const body = req.body as CreateDatasetCollectionParams;
const { userId } = await authUser({ req, authToken: true });
const body = req.body || {};
// auth. not visitor and dataset is public
const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true });
await authDataset({
req,
authToken: true,
datasetId: body.datasetId,
per: 'r'
});
jsonRes(res, {
data: await createOneCollection({
...body,
userId
teamId,
tmbId
})
});
} catch (err) {
@@ -39,11 +46,13 @@ export async function createOneCollection({
datasetId,
type,
metadata = {},
userId
}: CreateDatasetCollectionParams & { userId: string }) {
teamId,
tmbId
}: CreateDatasetCollectionParams & { teamId: string; tmbId: string }) {
const { _id } = await MongoDatasetCollection.create({
name,
userId,
teamId,
tmbId,
datasetId,
parentId: parentId || null,
type,
@@ -56,7 +65,8 @@ export async function createOneCollection({
await createDefaultCollection({
datasetId,
parentId: _id,
userId
teamId,
tmbId
});
}
@@ -68,20 +78,23 @@ export function createDefaultCollection({
name = '手动录入',
datasetId,
parentId,
userId
teamId,
tmbId
}: {
name?: '手动录入' | '手动标注';
datasetId: string;
parentId?: string;
userId: string;
teamId: string;
tmbId: string;
}) {
return MongoDatasetCollection.create({
name,
userId,
teamId,
tmbId,
datasetId,
parentId,
type: DatasetCollectionTypeEnum.virtual,
updateTime: new Date('2000'),
updateTime: new Date('2099'),
metadata: {}
});
}

38
projects/app/src/pages/api/core/dataset/collection/delById.ts
View File

@@ -1,12 +1,13 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils';
import { delDataByCollectionId } from '@/service/core/dataset/data/utils';
import { delDataByCollectionId } from '@/service/core/dataset/data/controller';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { GridFSStorage } from '@/service/lib/gridfs';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { delFileById } from '@fastgpt/service/common/file/gridfs/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -18,37 +19,40 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('CollectionIdId is required');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { teamId } = await authDatasetCollection({
req,
authToken: true,
collectionId,
per: 'w'
});
// find all delete id
const collections = await findCollectionAndChild(collectionId, '_id metadata');
const delIdList = collections.map((item) => item._id);
// delete pg data
await delDataByCollectionId({ userId, collectionIds: delIdList });
await delDataByCollectionId({ collectionIds: delIdList });
// delete training data
await MongoDatasetTraining.deleteMany({
datasetCollectionId: { $in: delIdList },
userId
teamId
});
// delete file
const gridFs = new GridFSStorage('dataset', userId);
const fileCollection = gridFs.Collection();
await Promise.all(
collections.map(
(item) =>
//@ts-ignore
item.metadata?.fileId && fileCollection.findOneAndDelete({ _id: item.metadata.fileId })
)
collections.map((collection) => {
if (!collection.metadata?.fileId) return;
return delFileById({
bucketName: BucketNameEnum.dataset,
fileId: collection.metadata.fileId
});
})
);
// delete collection
await MongoDatasetCollection.deleteMany({
_id: { $in: delIdList },
userId
_id: { $in: delIdList }
});
jsonRes(res);

27
projects/app/src/pages/api/core/dataset/collection/detail.ts
View File

@@ -2,10 +2,10 @@
Get one dataset collection detail
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -17,16 +17,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { collection, canWrite } = await authDatasetCollection({
req,
authToken: true,
collectionId: id,
per: 'r'
});
const collection = await MongoDatasetCollection.findOne({ _id: id, userId }).lean();
if (!collection) {
throw new Error('Collection not found');
}
jsonRes(res, {
data: collection
jsonRes<DatasetCollectionItemType>(res, {
data: {
...collection,
datasetId: collection.datasetId._id,
canWrite
}
});
} catch (err) {
jsonRes(res, {

26
projects/app/src/pages/api/core/dataset/collection/list.ts
View File

@@ -1,9 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { Types } from '@fastgpt/service/common/mongo';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/response';
import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
@@ -12,6 +10,8 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection
import { countCollectionData } from '@/service/core/dataset/data/utils';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { startQueue } from '@/service/utils/tools';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { getTeamInfoByTmbId } from '@fastgpt/service/support/user/team/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -28,11 +28,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
} = req.body as GetDatasetCollectionsProps;
searchText = searchText?.replace(/'/g, '');
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
// auth dataset and get my role
const { tmbId } = await authDataset({ req, authToken: true, datasetId, per: 'r' });
const { canWrite } = await getTeamInfoByTmbId({ tmbId });
const match = {
userId: new Types.ObjectId(userId),
datasetId: new Types.ObjectId(datasetId),
parentId: parentId ? new Types.ObjectId(parentId) : null,
...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}),
@@ -43,6 +43,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
: {})
};
// not count data amount
if (simple) {
const collections = await MongoDatasetCollection.find(match, '_id name type parentId')
.sort({
@@ -57,7 +58,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
collections.map(async (item) => ({
...item,
dataAmount: 0,
trainingAmount: 0
trainingAmount: 0,
canWrite // admin or owner can write
}))
),
total: await MongoDatasetCollection.countDocuments(match)
@@ -65,7 +67,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
}
const collections = await MongoDatasetCollection.aggregate([
const collections: DatasetCollectionsListItemType[] = await MongoDatasetCollection.aggregate([
{
$match: match
},
@@ -82,11 +84,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
$project: {
_id: 1,
parentId: 1,
fileId: 1,
tmbId: 1,
name: 1,
type: 1,
updateTime: 1,
trainingAmount: { $size: '$trainings_amount' }
trainingAmount: { $size: '$trainings_amount' },
metadata: 1
}
},
{
@@ -108,7 +111,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const data = await Promise.all(
collections.map(async (item, i) => ({
...item,
dataAmount: item.type === DatasetCollectionTypeEnum.folder ? undefined : counts[i]
dataAmount: item.type === DatasetCollectionTypeEnum.folder ? undefined : counts[i],
canWrite: String(item.tmbId) === tmbId || canWrite
}))
);

15
projects/app/src/pages/api/core/dataset/collection/paths.ts
View File

@@ -1,9 +1,9 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { DatasetPathItemType } from '@/types/core/dataset';
import { getDatasetCollectionPaths } from '@fastgpt/service/core/dataset/collection/utils';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -11,10 +11,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { parentId } = req.query as { parentId: string };
const { userId } = await authUser({ req, authToken: true });
if (!parentId) {
return jsonRes(res, {
data: []
});
}
await authDatasetCollection({ req, authToken: true, collectionId: parentId, per: 'r' });
const paths = await getDatasetCollectionPaths({
parentId,
userId
parentId
});
jsonRes<DatasetPathItemType[]>(res, {

18
projects/app/src/pages/api/core/dataset/collection/update.ts
View File

@@ -1,10 +1,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import type { UpdateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -16,7 +16,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await authDatasetCollection({ req, authToken: true, collectionId: id, per: 'w' });
const updateFields: Record<string, any> = {
...(parentId !== undefined && { parentId: parentId || null }),
@@ -28,15 +28,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
updateFields[`metadata.${key}`] = value;
}
await MongoDatasetCollection.findOneAndUpdate(
{
_id: id,
userId
},
{
$set: updateFields
}
);
await MongoDatasetCollection.findByIdAndUpdate(id, {
$set: updateFields
});
jsonRes(res);
} catch (err) {

12
projects/app/src/pages/api/core/dataset/create.ts
View File

@@ -1,10 +1,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import type { CreateDatasetParams } from '@/global/core/api/datasetReq.d';
import { createDefaultCollection } from './collection/create';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -12,11 +12,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { name, tags, avatar, vectorModel, parentId, type } = req.body as CreateDatasetParams;
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true });
const { _id } = await MongoDataset.create({
name,
userId,
teamId,
tmbId,
tags,
vectorModel,
avatar,
@@ -26,7 +27,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await createDefaultCollection({
datasetId: _id,
userId
teamId,
tmbId
});
jsonRes(res, { data: _id });

12
projects/app/src/pages/api/core/dataset/data/delDataById.ts
View File

@@ -1,10 +1,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { PgClient } from '@/service/pg';
import { jsonRes } from '@fastgpt/service/common/response';
import { PgClient } from '@fastgpt/service/common/pg';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { PgDatasetTableName } from '@/constants/plugin';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -18,10 +18,10 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await authDatasetData({ req, authToken: true, dataId, per: 'w' });
await PgClient.delete(PgDatasetTableName, {
where: [['user_id', userId], 'AND', ['id', dataId]]
where: [['id', dataId]]
});
jsonRes(res);

20
projects/app/src/pages/api/core/dataset/data/exportAll.ts
View File

@@ -1,14 +1,14 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { PgDatasetTableName } from '@/constants/plugin';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { findAllChildrenIds } from '../delete';
import QueryStream from 'pg-query-stream';
import { PgClient } from '@/service/pg';
import { addLog } from '@/service/utils/tools';
import { PgClient, Pg } from '@fastgpt/service/common/pg';
import { addLog } from '@fastgpt/service/common/mongo/controller';
import { responseWriteController } from '@fastgpt/service/common/response';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -17,12 +17,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
datasetId: string;
};
if (!datasetId || !global.pgClient) {
if (!datasetId || !Pg) {
throw new Error('缺少参数');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { userId } = await authDataset({ req, authToken: true, datasetId, per: 'w' });
const exportIds = [datasetId, ...(await findAllChildrenIds(datasetId))];
@@ -48,7 +48,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
const { rows } = await PgClient.query(
`SELECT count(id) FROM ${PgDatasetTableName} where user_id='${userId}' AND dataset_id IN (${exportIds
`SELECT count(id) FROM ${PgDatasetTableName} where dataset_id IN (${exportIds
.map((id) => `'${id}'`)
.join(',')})`
);
@@ -61,7 +61,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// connect pg
global.pgClient.connect((err, client, done) => {
Pg.connect((err, client, done) => {
if (err) {
console.error(err);
res.end('Error connecting to database');
@@ -71,7 +71,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// create pg select stream
const query = new QueryStream(
`SELECT q, a FROM ${PgDatasetTableName} where user_id='${userId}' AND dataset_id IN (${exportIds
`SELECT q, a FROM ${PgDatasetTableName} where dataset_id IN (${exportIds
.map((id) => `'${id}'`)
.join(',')})`
);

79
projects/app/src/pages/api/core/dataset/data/getDataById.ts
View File

@@ -1,11 +1,7 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import type { DatasetDataItemType, PgDataItemType } from '@fastgpt/global/core/dataset/type';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
export type Response = {
id: string;
@@ -25,10 +21,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { datasetData } = await authDatasetData({ req, authToken: true, dataId, per: 'r' });
jsonRes(res, {
data: await getDatasetDataById({ userId, id: dataId })
data: datasetData
});
} catch (err) {
jsonRes(res, {
@@ -37,70 +33,3 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
}
}
export async function getDatasetDataById({
id,
userId
}: {
id: string;
userId: string;
}): Promise<DatasetDataItemType> {
const where: any = [['user_id', userId], 'AND', ['id', id]];
const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, {
fields: ['id', 'q', 'a', 'dataset_id', 'collection_id'],
where,
limit: 1
});
const data = searchRes?.rows?.[0];
if (!data) {
return Promise.reject('Data not found');
}
// find source
const collection = (await getDatasetDataItemInfo({ pgDataList: [data] }))[0];
if (!collection) {
return Promise.reject('Data Collection not found');
}
return {
id: data.id,
q: data.q,
a: data.a,
datasetId: data.dataset_id,
collectionId: data.collection_id,
sourceName: collection.sourceName,
sourceId: collection.sourceId
};
}
export async function getDatasetDataItemInfo({
pgDataList
}: {
pgDataList: PgDataItemType[];
}): Promise<DatasetDataItemType[]> {
const collections = await MongoDatasetCollection.find(
{
_id: { $in: pgDataList.map((item) => item.collection_id) }
},
'_id name datasetId metadata'
).lean();
return pgDataList.map((item) => {
const collection = collections.find(
(collection) => String(collection._id) === item.collection_id
);
return {
id: item.id,
q: item.q,
a: item.a,
datasetId: collection?.datasetId || '',
collectionId: item.collection_id,
sourceName: collection?.name || '',
sourceId: collection?.metadata?.fileId || collection?.metadata?.rawLink
};
});
}

15
projects/app/src/pages/api/core/dataset/data/getDataList.ts
View File

@@ -1,11 +1,11 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { PgClient } from '@fastgpt/service/common/pg';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import type { DatasetDataListItemType } from '@/global/core/dataset/response.d';
import type { GetDatasetDataListProps } from '@/global/core/api/datasetReq';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -16,18 +16,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
searchText = '',
collectionId
} = req.body as GetDatasetDataListProps;
if (!collectionId) {
throw new Error('collectionId is required');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await authDatasetCollection({ req, authToken: true, collectionId, per: 'r' });
searchText = searchText.replace(/'/g, '');
const where: any = [
['user_id', userId],
'AND',
['collection_id', collectionId],
searchText ? `AND (q ILIKE '%${searchText}%' OR a ILIKE '%${searchText}%')` : ''
];

6
projects/app/src/pages/api/core/dataset/data/getQueueLen.ts
View File

@@ -1,14 +1,14 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
/* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authToken: true });
await authCert({ req, authToken: true });
// split queue data
const result = await MongoDatasetTraining.countDocuments({

137
projects/app/src/pages/api/core/dataset/data/insertData.ts
View File

@@ -3,30 +3,87 @@
manual input or mark data
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { SetOneDatasetDataProps } from '@/global/core/api/datasetReq';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import { countPromptTokens } from '@/global/common/tiktoken';
import { getVectorModel } from '@/service/core/ai/model';
import { insertData2Dataset, hasSameValue } from '@/service/core/dataset/data/utils';
import { hasSameValue } from '@/service/core/dataset/data/utils';
import { insertData2Dataset } from '@/service/core/dataset/data/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { collectionId, q, a } = req.body as SetOneDatasetDataProps;
if (!q) {
return Promise.reject('q is required');
}
if (!collectionId) {
return Promise.reject('collectionId is required');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { teamId, tmbId } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'w'
});
await authTeamBalance(teamId);
// auth collection and get dataset
const [
{
datasetId: { _id: datasetId, vectorModel }
}
] = await Promise.all([getCollectionWithDataset(collectionId), authTeamBalance(teamId)]);
// format data
const formatQ = q.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
const formatA = a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"') || '';
// token check
const token = countPromptTokens(formatQ, 'system');
if (token > getVectorModel(vectorModel).maxToken) {
return Promise.reject('Q Over Tokens');
}
// Duplicate data check
await hasSameValue({
collectionId,
q: formatQ,
a: formatA
});
const { insertId, tokenLen } = await insertData2Dataset({
teamId,
tmbId,
q: formatQ,
a: formatA,
collectionId,
datasetId,
model: vectorModel
});
pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
model: vectorModel
});
jsonRes<string>(res, {
data: await getVectorAndInsertDataset({
...req.body,
userId
})
data: insertId
});
} catch (err) {
jsonRes(res, {
@@ -35,61 +92,3 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
}
});
export async function getVectorAndInsertDataset(
props: SetOneDatasetDataProps & { userId: string }
): Promise<string> {
let { datasetId, collectionId, q, a, userId } = props;
if (!datasetId) {
return Promise.reject('知识库 ID 不能为空');
}
if (!q) {
return Promise.reject('索引内容不能为空');
}
if (!collectionId) {
return Promise.reject('集合 ID 和集合类型不能同时为空');
}
// auth collection and get dataset
const collection = await MongoDatasetCollection.findOne({
_id: collectionId,
userId,
datasetId,
type: { $ne: DatasetCollectionTypeEnum.folder }
}).populate('datasetId', '_id vectorModel');
if (!collection) {
return Promise.reject('集合不存在');
}
const dataset = collection.datasetId as unknown as DatasetSchemaType;
// format data
const formatQ = q?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
const formatA = a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"') || '';
// token check
const token = countPromptTokens(formatQ, 'system');
if (token > getVectorModel(dataset.vectorModel).maxToken) {
return Promise.reject('Q Over Tokens');
}
// Duplicate data check
await hasSameValue({
collectionId,
q,
a
});
return insertData2Dataset({
userId,
q: formatQ,
a: formatA,
collectionId,
datasetId,
model: dataset.vectorModel
});
}

37
projects/app/src/pages/api/core/dataset/data/pushData.ts
View File

@@ -1,10 +1,8 @@
/* push data to training queue */
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authCollection } from '@fastgpt/service/core/dataset/auth';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { startQueue } from '@/service/utils/tools';
@@ -13,6 +11,8 @@ import { countPromptTokens } from '@/global/common/tiktoken';
import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
import type { PushDataProps } from '@/global/core/api/datasetReq.d';
import { getVectorModel } from '@/service/core/ai/model';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
const modeMap = {
[TrainingModeEnum.index]: true,
@@ -37,12 +37,19 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true, authApiKey: true });
const { teamId, tmbId } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'w'
});
jsonRes<PushDataResponse>(res, {
data: await pushDataToDatasetCollection({
...req.body,
userId
teamId,
tmbId
})
});
} catch (err) {
@@ -54,20 +61,19 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
export async function pushDataToDatasetCollection({
userId,
teamId,
tmbId,
collectionId,
data,
mode,
prompt,
billId
}: { userId: string } & PushDataProps): Promise<PushDataResponse> {
// auth dataset & get training model
}: { teamId: string; tmbId: string } & PushDataProps): Promise<PushDataResponse> {
// get vector model
const {
dataset: { _id: datasetId, vectorModel }
} = await authCollection({
userId,
collectionId
});
datasetId: { _id: datasetId, vectorModel }
} = await getCollectionWithDataset(collectionId);
const vectorModelData = getVectorModel(vectorModel);
const modeMap = {
@@ -76,7 +82,7 @@ export async function pushDataToDatasetCollection({
model: vectorModelData.model
},
[TrainingModeEnum.qa]: {
maxToken: global.qaModels[0].maxToken * 0.8,
maxToken: global.qaModels[0].maxContext * 0.8,
model: global.qaModels[0].model
}
};
@@ -119,7 +125,8 @@ export async function pushDataToDatasetCollection({
// 插入记录
const insertRes = await MongoDatasetTraining.insertMany(
filterResult.success.map((item) => ({
userId,
teamId,
tmbId,
datasetId,
datasetCollectionId: collectionId,
billId,

35
projects/app/src/pages/api/core/dataset/data/updateData.ts
View File

@@ -1,39 +1,54 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { SetOneDatasetDataProps } from '@/global/core/api/datasetReq.d';
import { updateData2Dataset } from '@/service/core/dataset/data/utils';
import { updateData2Dataset } from '@/service/core/dataset/data/controller';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, datasetId, collectionId, q = '', a } = req.body as SetOneDatasetDataProps;
const { id, collectionId, q = '', a } = req.body as SetOneDatasetDataProps;
if (!id || !collectionId) {
throw new Error('缺少参数');
}
// auth data permission
const { datasetData, teamId, tmbId } = await authDatasetData({
req,
authToken: true,
dataId: id,
per: 'w'
});
// auth team balance
await authTeamBalance(teamId);
// auth user and get kb
const [{ userId }, dataset] = await Promise.all([
authUser({ req, authToken: true }),
MongoDataset.findById(datasetId, 'vectorModel')
]);
const dataset = await MongoDataset.findById(datasetData.datasetId, 'vectorModel');
if (!dataset) {
throw new Error("Can't find database");
}
await updateData2Dataset({
const { tokenLen } = await updateData2Dataset({
dataId: id,
userId,
q,
a,
model: dataset.vectorModel
});
pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
model: dataset.vectorModel
});
jsonRes(res);
} catch (err) {
jsonRes(res, {

27
projects/app/src/pages/api/core/dataset/delete.ts
View File

@@ -1,14 +1,14 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { GridFSStorage } from '@/service/lib/gridfs';
import { PgClient } from '@fastgpt/service/common/pg';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { delDatasetFiles } from '@fastgpt/service/core/dataset/file/controller';
import { Types } from '@fastgpt/service/common/mongo';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -21,29 +21,23 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('缺少参数');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
// auth owner
await authDataset({ req, authToken: true, datasetId: id, per: 'owner' });
const deletedIds = [id, ...(await findAllChildrenIds(id))];
// delete training data
await MongoDatasetTraining.deleteMany({
userId,
datasetId: { $in: deletedIds.map((id) => new Types.ObjectId(id)) }
});
// delete all pg data
await PgClient.delete(PgDatasetTableName, {
where: [
['user_id', userId],
'AND',
`dataset_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`
]
where: [`dataset_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`]
});
// delete related files
const gridFs = new GridFSStorage('dataset', userId);
await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByDatasetId(id)));
await delDatasetFiles({ datasetId: id });
// delete collections
await MongoDatasetCollection.deleteMany({
@@ -52,8 +46,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// delete dataset data
await MongoDataset.deleteMany({
_id: { $in: deletedIds },
userId
_id: { $in: deletedIds }
});
jsonRes(res);

37
projects/app/src/pages/api/core/dataset/detail.ts
View File

@@ -1,41 +1,36 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { getVectorModel } from '@/service/core/ai/model';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { DatasetItemType } from '@/types/core/dataset';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id } = req.query as {
const { id: datasetId } = req.query as {
id: string;
};
if (!id) {
if (!datasetId) {
throw new Error('缺少参数');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const data = await MongoDataset.findOne({
_id: id,
userId
const { dataset, canWrite, isOwner } = await authDataset({
req,
authToken: true,
datasetId,
per: 'r'
});
if (!data) {
throw new Error('kb is not exist');
}
jsonRes(res, {
jsonRes<DatasetItemType>(res, {
data: {
_id: data._id,
avatar: data.avatar,
name: data.name,
userId: data.userId,
vectorModel: getVectorModel(data.vectorModel),
tags: data.tags.join(' ')
...dataset,
tags: dataset.tags.join(' '),
vectorModel: getVectorModel(dataset.vectorModel),
canWrite,
isOwner
}
});
} catch (err) {

31
projects/app/src/pages/api/core/dataset/file/delEmptyFiles.ts
View File

@@ -1,31 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { datasetId } = req.query as { datasetId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const collection = gridFs.Collection();
const files = await collection.deleteMany({
uploadDate: { $lte: new Date(Date.now() - 7 * 24 * 60 * 60 * 1000) },
['metadata.datasetId']: datasetId,
['metadata.userId']: userId,
['metadata.datasetUsed']: { $ne: true }
});
jsonRes(res, {
data: files
});
} catch (err) {
jsonRes(res);
}
}

53
projects/app/src/pages/api/core/dataset/file/detail.ts
View File

@@ -1,14 +1,8 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { datasetSpecialIdMap } from '@fastgpt/global/core/dataset/constant';
import { datasetSpecialIds } from '@fastgpt/global/core/dataset/constant';
import type { GSFileInfoType } from '@/types/common/file';
import { strIsLink } from '@fastgpt/global/common/string/tools';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { authDatasetFile } from '@fastgpt/service/support/permission/auth/dataset';
import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -16,46 +10,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { fileId } = req.query as { fileId: string };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { file } = await authDatasetFile({ req, authToken: true, fileId, per: 'r' });
// manual, mark
if (datasetSpecialIds.includes(fileId)) {
return jsonRes<GSFileInfoType>(res, {
data: {
id: fileId,
size: 0,
// @ts-ignore
filename: datasetSpecialIdMap[fileId]?.name || fileId,
uploadDate: new Date(),
encoding: '',
contentType: ''
}
});
}
// link file
if (strIsLink(fileId)) {
const { rows } = await PgClient.select(PgDatasetTableName, {
where: [['user_id', userId], 'AND', ['file_id', fileId]],
limit: 1,
fields: ['source']
});
return jsonRes<GSFileInfoType>(res, {
data: {
id: fileId,
size: 0,
filename: rows[0]?.source || fileId,
uploadDate: new Date(),
encoding: '',
contentType: ''
}
});
}
const gridFs = new GridFSStorage('dataset', userId);
const file = await gridFs.findAndAuthFile(fileId);
jsonRes<GSFileInfoType>(res, {
jsonRes<DatasetFileSchema>(res, {
data: file
});
} catch (err) {

36
projects/app/src/pages/api/core/dataset/file/getPreviewUrl.ts Normal file
View File

@@ -0,0 +1,36 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetFile } from '@fastgpt/service/support/permission/auth/dataset';
import { createFileToken } from '@fastgpt/service/support/permission/controller';
import { BucketNameEnum, FileBaseUrl } from '@fastgpt/global/common/file/constants';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { fileId } = req.query as { fileId: string };
if (!fileId) {
throw new Error('fileId is empty');
}
const { teamId, tmbId } = await authDatasetFile({ req, authToken: true, fileId, per: 'r' });
const token = await createFileToken({
bucketName: BucketNameEnum.dataset,
teamId,
tmbId,
fileId
});
jsonRes(res, {
data: `${FileBaseUrl}?token=${token}`
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}

70
projects/app/src/pages/api/core/dataset/file/update.ts
View File

@@ -1,70 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
import { Types } from '@fastgpt/service/common/mongo';
import { PgClient } from '@/service/pg';
import { PgDatasetTableName } from '@/constants/plugin';
import { addLog } from '@/service/utils/tools';
import { strIsLink } from '@fastgpt/global/common/string/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, name, datasetUsed } = req.body;
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
const collection = gridFs.Collection();
if (id.length === 24 && !strIsLink(id)) {
await collection.findOneAndUpdate(
{
_id: new Types.ObjectId(id)
},
{
$set: {
...(name && { filename: name }),
...(datasetUsed && { ['metadata.datasetUsed']: datasetUsed })
}
}
);
}
// data source
await updateDatasetSource({
fileId: id,
userId,
name
});
jsonRes(res, {});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
async function updateDatasetSource(data: { fileId: string; userId: string; name?: string }) {
const { fileId, userId, name } = data;
if (!fileId || !name || !userId) return;
try {
await PgClient.update(PgDatasetTableName, {
where: [['user_id', userId], 'AND', ['file_id', fileId]],
values: [
{
key: 'source',
value: name
}
]
});
} catch (error) {
addLog.error(`Update dataset source error`, error);
setTimeout(() => {
updateDatasetSource(data);
}, 2000);
}
}

21
projects/app/src/pages/api/core/dataset/list.ts
View File

@@ -1,21 +1,25 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { getVectorModel } from '@/service/core/ai/model';
import type { DatasetsItemType } from '@/types/core/dataset';
import type { DatasetItemType } from '@/types/core/dataset';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { parentId, type } = req.query as { parentId?: string; type?: `${DatasetTypeEnum}` };
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
const { teamId, tmbId, teamOwner, role, canWrite } = await authUserRole({
req,
authToken: true
});
const datasets = await MongoDataset.find({
userId,
...mongoRPermission({ teamId, tmbId, role }),
...(parentId !== undefined && { parentId: parentId || null }),
...(type && { type })
}).sort({ updateTime: -1 });
@@ -23,11 +27,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const data = await Promise.all(
datasets.map(async (item) => ({
...item.toJSON(),
vectorModel: getVectorModel(item.vectorModel)
tags: item.tags.join(' '),
vectorModel: getVectorModel(item.vectorModel),
canWrite,
isOwner: teamOwner || String(item.tmbId) === tmbId
}))
);
jsonRes<DatasetsItemType[]>(res, {
jsonRes<DatasetItemType[]>(res, {
data
});
} catch (err) {

11
projects/app/src/pages/api/core/dataset/paths.ts
View File

@@ -1,8 +1,9 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { DatasetPathItemType } from '@/types/core/dataset';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -10,6 +11,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const { parentId } = req.query as { parentId: string };
if (!parentId) {
return jsonRes(res, {
data: []
});
}
await authDataset({ req, authToken: true, datasetId: parentId, per: 'r' });
jsonRes<DatasetPathItemType[]>(res, {
data: await getParents(parentId)
});

87
projects/app/src/pages/api/core/dataset/searchTest.ts
View File

@@ -1,64 +1,67 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { PgClient } from '@/service/pg';
import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { getVector } from '../../openapi/plugin/vector';
import { PgDatasetTableName } from '@/constants/plugin';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { SearchTestProps } from '@/global/core/api/datasetReq.d';
import { connectToDatabase } from '@/service/mongo';
import type {
SearchDataResponseItemType,
SearchDataResultItemType
} from '@fastgpt/global/core/dataset/type';
import { getDatasetDataItemInfo } from './data/getDataById';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
import { countModelPrice } from '@/service/support/wallet/bill/utils';
import { searchDatasetData } from '@/service/core/dataset/data/utils';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
import { ModelTypeEnum } from '@/service/core/ai/model';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { datasetId, text } = req.body as SearchTestProps;
const { datasetId, text, limit = 20 } = req.body as SearchTestProps;
if (!datasetId || !text) {
throw new Error('缺少参数');
}
// 凭证校验
const [{ userId }, dataset] = await Promise.all([
authUser({ req, authToken: true, authApiKey: true }),
MongoDataset.findById(datasetId, 'vectorModel')
]);
if (!userId || !dataset) {
throw new Error('缺少用户ID');
}
const { vectors } = await getVector({
model: dataset.vectorModel,
userId,
input: [text]
// auth dataset role
const { dataset, teamId, tmbId, apikey } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'r'
});
const results: any = await PgClient.query(
`BEGIN;
SET LOCAL hnsw.ef_search = ${global.systemEnv.pgHNSWEfSearch || 100};
select id, q, a, dataset_id, collection_id, (vector <#> '[${
vectors[0]
}]') * -1 AS score from ${PgDatasetTableName} where dataset_id='${datasetId}' AND user_id='${userId}' ORDER BY vector <#> '[${
vectors[0]
}]' limit 12;
COMMIT;`
);
// auth balance
await authTeamBalance(teamId);
const rows = results?.[2]?.rows as SearchDataResultItemType[];
const { searchRes, tokenLen } = await searchDatasetData({
text,
model: dataset.vectorModel,
limit: Math.min(limit, 50),
datasetIds: [datasetId]
});
const collectionsData = await getDatasetDataItemInfo({ pgDataList: rows });
// push bill
pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
model: dataset.vectorModel,
source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt
});
if (apikey) {
updateApiKeyUsage({
apikey,
usage: countModelPrice({
model: dataset.vectorModel,
tokens: tokenLen,
type: ModelTypeEnum.vector
})
});
}
jsonRes<SearchDataResponseItemType[]>(res, {
data: collectionsData.map((item, index) => ({
...item,
score: rows[index].score
}))
data: searchRes
});
} catch (err) {
jsonRes(res, {

16
projects/app/src/pages/api/core/dataset/update.ts
View File

@@ -1,34 +1,32 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import type { DatasetUpdateParams } from '@/global/core/api/datasetReq.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id, parentId, name, avatar, tags } = req.body as DatasetUpdateParams;
const { id, parentId, name, avatar, tags, permission } = req.body as DatasetUpdateParams;
if (!id) {
throw new Error('缺少参数');
}
// 凭证校验
const { userId } = await authUser({ req, authToken: true });
await authDataset({ req, authToken: true, datasetId: id, per: 'owner' });
await MongoDataset.findOneAndUpdate(
{
_id: id,
userId
_id: id
},
{
...(parentId !== undefined && { parentId: parentId || null }),
...(name && { name }),
...(avatar && { avatar }),
...(typeof tags === 'string' && {
tags: tags.split(' ').filter((item) => item)
})
...(tags && { tags }),
...(permission && { permission })
}
);

17
projects/app/src/pages/api/core/plugin/create.ts
View File

@@ -1,19 +1,24 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { createOnePlugin } from '@fastgpt/service/core/plugin/controller';
import type { CreateOnePluginParams } from '@fastgpt/global/core/plugin/controller';
import { defaultModules } from '@fastgpt/global/core/plugin/constants';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true });
const body = req.body as CreateOnePluginParams;
const { _id } = await MongoPlugin.create({
...body,
teamId,
tmbId
});
jsonRes(res, {
data: await createOnePlugin({ userId, modules: defaultModules, ...body })
data: _id
});
} catch (err) {
jsonRes(res, {

14
projects/app/src/pages/api/core/plugin/delete.ts
View File

@@ -1,18 +1,18 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { deleteOnePlugin } from '@fastgpt/service/core/plugin/controller';
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
import { authPluginCrud } from '@fastgpt/service/support/permission/auth/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { id } = req.query as { id: string };
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
await authPluginCrud({ req, authToken: true, id, per: 'owner' });
jsonRes(res, {
data: await deleteOnePlugin({ id, userId })
});
await MongoPlugin.findByIdAndRemove(id);
jsonRes(res, {});
} catch (err) {
jsonRes(res, {
code: 500,

10
projects/app/src/pages/api/core/plugin/detail.ts
View File

@@ -1,17 +1,17 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { getOnePluginDetail } from '@fastgpt/service/core/plugin/controller';
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
import { authPluginCrud } from '@fastgpt/service/support/permission/auth/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { id } = req.query as { id: string };
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
await authPluginCrud({ req, authToken: true, id, per: 'r' });
jsonRes(res, {
data: await getOnePluginDetail({ id, userId })
data: await MongoPlugin.findOne({ id })
});
} catch (err) {
jsonRes(res, {

10
projects/app/src/pages/api/core/plugin/list.ts
View File

@@ -1,16 +1,16 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { getUserPlugins } from '@fastgpt/service/core/plugin/controller';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { teamId } = await authCert({ req, authToken: true });
jsonRes(res, {
data: await getUserPlugins({ userId })
data: await MongoPlugin.find({ teamId })
});
} catch (err) {
jsonRes(res, {

8
projects/app/src/pages/api/core/plugin/moduleDetail.ts
View File

@@ -1,17 +1,17 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { getPluginModuleDetail } from '@fastgpt/service/core/plugin/controller';
import { authPluginCrud } from '@fastgpt/service/support/permission/auth/plugin';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const { id } = req.query as { id: string };
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
await authPluginCrud({ req, authToken: true, id, per: 'r' });
jsonRes(res, {
data: await getPluginModuleDetail({ id, userId })
data: await getPluginModuleDetail({ id })
});
} catch (err) {
jsonRes(res, {

8
projects/app/src/pages/api/core/plugin/templateList.ts
View File

@@ -1,16 +1,16 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { getUserPlugins2Templates } from '@fastgpt/service/core/plugin/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { teamId } = await authCert({ req, authToken: true });
jsonRes(res, {
data: await getUserPlugins2Templates({ userId })
data: await getUserPlugins2Templates({ teamId })
});
} catch (err) {
jsonRes(res, {

13
projects/app/src/pages/api/core/plugin/update.ts
View File

@@ -1,18 +1,19 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { updateOnePlugin } from '@fastgpt/service/core/plugin/controller';
import type { UpdatePluginParams } from '@fastgpt/global/core/plugin/controller';
import { authPluginCrud } from '@fastgpt/service/support/permission/auth/plugin';
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const body = req.body as UpdatePluginParams;
const { id, ...props } = req.body as UpdatePluginParams;
await authPluginCrud({ req, authToken: true, id, per: 'owner' });
jsonRes(res, {
data: await updateOnePlugin({ userId, ...body })
data: await MongoPlugin.findByIdAndUpdate(id, props)
});
} catch (err) {
jsonRes(res, {

112
projects/app/src/pages/api/openapi/plugin/vector.ts
View File

@@ -1,112 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authBalanceByUid, authUser } from '@fastgpt/service/support/user/auth';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { getAIApi } from '@fastgpt/service/core/ai/config';
import { pushGenerateVectorBill } from '@/service/common/bill/push';
import { connectToDatabase } from '@/service/mongo';
type Props = {
model: string;
input: string[];
billId?: string;
};
type Response = {
tokenLen: number;
vectors: number[][];
};
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
let { input, model } = req.query as Props;
if (!Array.isArray(input)) {
throw new Error('缺少参数');
}
jsonRes<Response>(res, {
data: await getVector({ userId, input, model })
});
} catch (err) {
console.log(err);
jsonRes(res, {
code: 500,
error: err
});
}
});
export async function getVector({
model = 'text-embedding-ada-002',
userId,
input,
billId
}: { userId?: string } & Props) {
try {
userId && (await authBalanceByUid(userId));
for (let i = 0; i < input.length; i++) {
if (!input[i]) {
return Promise.reject({
code: 500,
message: '向量生成模块输入内容为空'
});
}
}
// 获取 chatAPI
const ai = getAIApi();
// 把输入的内容转成向量
const result = await ai.embeddings
.create(
{
model,
input
},
{
timeout: 60000
}
)
.then(async (res) => {
if (!res.data) {
return Promise.reject('Embedding API 404');
}
if (!res?.data?.[0]?.embedding) {
console.log(res?.data);
// @ts-ignore
return Promise.reject(res.data?.err?.message || 'Embedding API Error');
}
return {
tokenLen: res.usage.total_tokens || 0,
vectors: await Promise.all(res.data.map((item) => unityDimensional(item.embedding)))
};
});
userId &&
pushGenerateVectorBill({
userId,
tokenLen: result.tokenLen,
model,
billId
});
return result;
} catch (error) {
console.log(`Embedding Error`, error);
return Promise.reject(error);
}
}
function unityDimensional(vector: number[]) {
if (vector.length > 1536) return Promise.reject('向量维度不能超过 1536');
let resultVector = vector;
const vectorLen = vector.length;
const zeroVector = new Array(1536 - vectorLen).fill(0);
return resultVector.concat(zeroVector);
}

6
projects/app/src/pages/api/plugins/urlFetch.ts
View File

@@ -3,8 +3,8 @@ import { NextApiRequest, NextApiResponse } from 'next';
import axios from 'axios';
import { JSDOM } from 'jsdom';
import { Readability } from '@mozilla/readability';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import type { FetchResultItem } from '@fastgpt/global/common/plugin/types/pluginRes.d';
import { simpleText } from '@fastgpt/global/common/string/tools';
import { connectToDatabase } from '@/service/mongo';
@@ -20,7 +20,7 @@ const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
throw new Error('urlList is empty');
}
await authUser({ req, authToken: true });
await authCert({ req, authToken: true });
urlList = urlList.filter((url) => /^(http|https):\/\/[^ "]+$/.test(url));

13
projects/app/src/pages/api/plusApi/[...path].ts
View File

@@ -1,8 +1,9 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { request } from '@fastgpt/service/common/api/plusRequest';
import type { Method } from 'axios';
import { connectToDatabase } from '@/service/mongo';
import { setCookie } from '@fastgpt/service/support/permission/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -32,6 +33,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
method
);
/* special response */
// response cookie
if (repose?.cookie) {
setCookie(res, repose.cookie);
return jsonRes(res, {
data: repose?.cookie
});
}
jsonRes(res, {
data: repose
});

13
projects/app/src/pages/api/support/openapi/postKey.ts → projects/app/src/pages/api/support/openapi/create.ts
View File

@@ -1,19 +1,19 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { customAlphabet } from 'nanoid';
import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d';
import type { EditApiKeyProps } from '@/global/support/openapi/api';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId, name, limit } = req.body as EditApiKeyProps;
const { userId } = await authUser({ req, authToken: true });
const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true });
const count = await MongoOpenApi.find({ userId, appId }).countDocuments();
const count = await MongoOpenApi.find({ tmbId, appId }).countDocuments();
if (count >= 10) {
throw new Error('最多 10 组 API 秘钥');
@@ -26,7 +26,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const apiKey = `${global.systemEnv?.openapiPrefix || 'fastgpt'}-${nanoid()}`;
await MongoOpenApi.create({
userId,
teamId,
tmbId,
apiKey,
appId,
name,

8
projects/app/src/pages/api/support/openapi/delKey.ts → projects/app/src/pages/api/support/openapi/delete.ts
View File

@@ -1,9 +1,9 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authOpenApiKeyCrud } from '@fastgpt/service/support/permission/auth/openapi';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -14,9 +14,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('缺少参数');
}
const { userId } = await authUser({ req, authToken: true });
await authOpenApiKeyCrud({ req, authToken: true, id, per: 'owner' });
await MongoOpenApi.findOneAndRemove({ _id: id, userId });
await MongoOpenApi.findOneAndRemove({ _id: id });
jsonRes(res);
} catch (err) {

25
projects/app/src/pages/api/support/openapi/getKeys.ts
View File

@@ -1,25 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import type { GetApiKeyProps } from '@/global/support/api/openapiReq.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId } = req.query as GetApiKeyProps;
const { userId } = await authUser({ req, authToken: true });
const findResponse = await MongoOpenApi.find({ userId, appId }).sort({ _id: -1 });
jsonRes(res, {
data: findResponse.map((item) => item.toObject())
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

48
projects/app/src/pages/api/support/openapi/list.ts Normal file
View File

@@ -0,0 +1,48 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
import type { GetApiKeyProps } from '@/global/support/openapi/api';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId } = req.query as GetApiKeyProps;
if (appId) {
const { tmbId, teamOwner } = await authApp({ req, authToken: true, appId, per: 'w' });
const findResponse = await MongoOpenApi.find({
appId,
...(!teamOwner && { tmbId })
}).sort({ _id: -1 });
return jsonRes(res, {
data: findResponse.map((item) => item.toObject())
});
}
const {
teamId,
tmbId,
isOwner: teamOwner
} = await authUserNotVisitor({ req, authToken: true });
const findResponse = await MongoOpenApi.find({
appId,
teamId,
...(!teamOwner && { tmbId })
}).sort({ _id: -1 });
return jsonRes(res, {
data: findResponse.map((item) => item.toObject())
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

23
projects/app/src/pages/api/support/openapi/putKey.ts → projects/app/src/pages/api/support/openapi/update.ts
View File

@@ -1,26 +1,21 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import type { EditApiKeyProps } from '@/global/support/api/openapiReq.d';
import type { EditApiKeyProps } from '@/global/support/openapi/api.d';
import { authOpenApiKeyCrud } from '@fastgpt/service/support/permission/auth/openapi';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { _id, name, limit } = req.body as EditApiKeyProps & { _id: string };
const { userId } = await authUser({ req, authToken: true });
await MongoOpenApi.findOneAndUpdate(
{
_id,
userId
},
{
...(name && { name }),
...(limit && { limit })
}
);
await authOpenApiKeyCrud({ req, authToken: true, id: _id, per: 'owner' });
await MongoOpenApi.findByIdAndUpdate(_id, {
...(name && { name }),
...(limit && { limit })
});
jsonRes(res);
} catch (err) {

15
projects/app/src/pages/api/support/outLink/create.ts
View File

@@ -1,9 +1,8 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { authApp } from '@/service/utils/auth';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import type { OutLinkEditType } from '@fastgpt/global/support/outLink/type.d';
import { customAlphabet } from 'nanoid';
import { OutLinkTypeEnum } from '@fastgpt/global/support/outLink/constant';
@@ -18,17 +17,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
type: `${OutLinkTypeEnum}`;
};
const { userId } = await authUser({ req, authToken: true });
await authApp({
appId,
userId,
authOwner: false
});
const { teamId, tmbId } = await authApp({ req, authToken: true, appId, per: 'w' });
const shareId = nanoid();
await MongoOutLink.create({
shareId,
userId,
teamId,
tmbId,
appId,
...props
});

11
projects/app/src/pages/api/support/outLink/delete.ts
View File

@@ -1,8 +1,8 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authOutLinkCrud } from '@fastgpt/service/support/permission/auth/outLink';
/* delete a shareChat by shareChatId */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -13,12 +13,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
id: string;
};
const { userId } = await authUser({ req, authToken: true });
await authOutLinkCrud({ req, outLinkId: id, authToken: true, per: 'owner' });
await MongoOutLink.findOneAndRemove({
_id: id,
userId
});
await MongoOutLink.findByIdAndRemove(id);
jsonRes(res);
} catch (err) {

31
projects/app/src/pages/api/support/outLink/init.ts
View File

@@ -1,14 +1,13 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import type { InitShareChatResponse } from '@/global/support/api/outLinkRes.d';
import { authApp } from '@/service/utils/auth';
import { HUMAN_ICON } from '@/constants/chat';
import type { InitShareChatResponse } from '@fastgpt/global/support/outLink/api.d';
import { HUMAN_ICON } from '@fastgpt/global/core/chat/constants';
import { getGuideModule } from '@/global/core/app/modules/utils';
import { authShareChatInit } from '@fastgpt/service/support/outLink/auth';
import { authShareChatInit } from '@/service/support/outLink/auth';
import { getChatModelNameListByModules } from '@/service/core/app/module';
import { authOutLinkValid } from '@fastgpt/service/support/permission/auth/outLink';
/* init share chat window */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -19,27 +18,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
authToken?: string;
};
if (!shareId) {
throw new Error('params is error');
}
// get shareChat
const shareChat = await MongoOutLink.findOne({ shareId });
if (!shareChat) {
return jsonRes(res, {
code: 501,
error: '分享链接已失效'
});
}
const { app, shareChat } = await authOutLinkValid({ shareId });
// 校验使用权限
const [{ app }, user] = await Promise.all([
authApp({
appId: shareChat.appId,
userId: String(shareChat.userId),
authOwner: false
}),
const [user] = await Promise.all([
MongoUser.findById(shareChat.userId, 'avatar'),
authShareChatInit({
authToken,

8
projects/app/src/pages/api/support/outLink/list.ts
View File

@@ -1,8 +1,8 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
/* get shareChat list by appId */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -13,11 +13,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
appId: string;
};
const { userId } = await authUser({ req, authToken: true });
const { teamId, tmbId, isOwner } = await authApp({ req, authToken: true, appId, per: 'w' });
const data = await MongoOutLink.find({
appId,
userId
...(isOwner ? { teamId } : { tmbId })
}).sort({
_id: -1
});

9
projects/app/src/pages/api/support/outLink/update.ts
View File

@@ -1,8 +1,9 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import type { OutLinkEditType } from '@fastgpt/global/support/outLink/type.d';
import { authOutLinkCrud } from '@fastgpt/service/support/permission/auth/outLink';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -10,6 +11,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { _id, name, responseDetail, limit } = req.body as OutLinkEditType & {};
if (!_id) {
throw new Error('_id is required');
}
await authOutLinkCrud({ req, outLinkId: _id, authToken: true, per: 'owner' });
await MongoOutLink.findByIdAndUpdate(_id, {
name,
responseDetail,

20
projects/app/src/pages/api/common/bill/createTrainingBill.ts → projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts
View File

@@ -1,21 +1,23 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase, Bill } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { BillSourceEnum } from '@/constants/user';
import { CreateTrainingBillType } from '@fastgpt/global/common/bill/types/billReq.d';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { CreateTrainingBillProps } from '@fastgpt/global/support/wallet/bill/api.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { name } = req.body as CreateTrainingBillType;
const { name } = req.body as CreateTrainingBillProps;
const { userId } = await authUser({ req, authToken: true, authApiKey: true });
const { teamId, tmbId } = await authCert({ req, authToken: true, authApiKey: true });
const qaModel = global.qaModels[0];
const { _id } = await Bill.create({
userId,
const { _id } = await MongoBill.create({
teamId,
tmbId,
appName: name,
source: BillSourceEnum.training,
list: [

30
projects/app/src/pages/api/system/file/delete.ts
View File

@@ -1,30 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { fileId } = req.query as { fileId: string };
if (!fileId) {
throw new Error('fileId is empty');
}
const { userId } = await authUser({ req, authToken: true });
const gridFs = new GridFSStorage('dataset', userId);
await gridFs.delete(fileId);
jsonRes(res);
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}

75
projects/app/src/pages/api/system/file/readUrl.ts
View File

@@ -1,75 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import jwt from 'jsonwebtoken';
import { ERROR_ENUM } from '@fastgpt/global/common/error/errorCode';
import { GridFSStorage } from '@/service/lib/gridfs';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { fileId } = req.query as { fileId: string };
if (!fileId) {
throw new Error('fileId is empty');
}
const { userId } = await authUser({ req, authToken: true });
// auth file
const gridFs = new GridFSStorage('dataset', userId);
await gridFs.findAndAuthFile(fileId);
const token = await createFileToken({
userId,
fileId
});
jsonRes(res, {
data: `/api/system/file/read?token=${token}`
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
export const createFileToken = (data: { userId: string; fileId: string }) => {
if (!process.env.FILE_TOKEN_KEY) {
return Promise.reject('System unset FILE_TOKEN_KEY');
}
const expiredTime = Math.floor(Date.now() / 1000) + 60 * 30;
const key = process.env.FILE_TOKEN_KEY as string;
const token = jwt.sign(
{
...data,
exp: expiredTime
},
key
);
return Promise.resolve(token);
};
export const authFileToken = (token?: string) =>
new Promise<{ userId: string; fileId: string }>((resolve, reject) => {
if (!token) {
return reject(ERROR_ENUM.unAuthFile);
}
const key = process.env.FILE_TOKEN_KEY as string;
jwt.verify(token, key, function (err, decoded: any) {
if (err || !decoded?.userId || !decoded?.fileId) {
reject(ERROR_ENUM.unAuthFile);
return;
}
resolve({
userId: decoded.userId,
fileId: decoded.fileId
});
});
});

30
projects/app/src/pages/api/system/getInitData.ts
View File

@@ -1,9 +1,9 @@
import type { FeConfigsType, SystemEnvType } from '@fastgpt/global/common/system/types/index.d';
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { readFileSync } from 'fs';
import type { InitDateResponse } from '@/global/common/api/systemRes';
import { formatPrice } from '@fastgpt/global/common/bill/tools';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import { getTikTokenEnc } from '@/global/common/tiktoken';
import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent';
import {
@@ -12,14 +12,16 @@ import {
defaultCQModels,
defaultExtractModels,
defaultQGModels,
defaultVectorModels
} from '@/constants/model';
defaultVectorModels,
defaultAudioSpeechModels
} from '@fastgpt/global/core/ai/model';
import {
AudioSpeechModelType,
ChatModelItemType,
FunctionModelItemType,
LLMModelItemType,
VectorModelItemType
} from '@/types/model';
} from '@fastgpt/global/core/ai/model.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
getInitConfig();
@@ -66,8 +68,6 @@ export function initGlobal() {
initHttpAgent();
global.qaQueueLen = 0;
global.vectorQueueLen = 0;
global.sendInformQueue = [];
global.sendInformQueueLen = 0;
}
export function getInitConfig() {
@@ -87,6 +87,7 @@ export function getInitConfig() {
ExtractModels: FunctionModelItemType[];
QGModels: LLMModelItemType[];
VectorModels: VectorModelItemType[];
AudioSpeechModels: AudioSpeechModelType[];
};
console.log(`System Version: ${global.systemVersion}`);
@@ -105,6 +106,8 @@ export function getInitConfig() {
global.qgModels = res.QGModels || defaultQGModels;
global.vectorModels = res.VectorModels || defaultVectorModels;
global.audioSpeechModels = res.AudioSpeechModels || defaultAudioSpeechModels;
} catch (error) {
setDefaultData();
console.log('get init config error, set default', error);
@@ -122,19 +125,12 @@ export function setDefaultData() {
global.qgModels = defaultQGModels;
global.vectorModels = defaultVectorModels;
global.audioSpeechModels = defaultAudioSpeechModels;
global.priceMd = '';
console.log('use default config');
console.log({
feConfigs: defaultFeConfigs,
systemEnv: defaultSystemEnv,
chatModels: defaultChatModels,
qaModels: defaultQAModels,
cqModels: defaultCQModels,
extractModels: defaultExtractModels,
qgModels: defaultQGModels,
vectorModels: defaultVectorModels
});
console.log(global);
}
export function getSystemVersion() {

2
projects/app/src/pages/api/system/img/[id].ts
View File

@@ -1,5 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { readMongoImg } from '@fastgpt/service/common/file/image/controller';

19
projects/app/src/pages/api/user/account/loginByPassword.ts
View File

@@ -1,25 +1,26 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { setCookie } from '@fastgpt/service/support/user/auth';
import { generateToken } from '@fastgpt/service/support/user/auth';
import { createJWT, setCookie } from '@fastgpt/service/support/permission/controller';
import { connectToDatabase } from '@/service/mongo';
import { getUserDetail } from '@/service/support/user/controller';
import type { PostLoginProps } from '@fastgpt/global/support/user/api.d';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { username, password } = req.body;
const { username, password, tmbId = '' } = req.body as PostLoginProps;
if (!username || !password) {
throw new Error('缺少参数');
}
// 检测用户是否存在
const authUser = await MongoUser.findOne({
const authCert = await MongoUser.findOne({
username
});
if (!authUser) {
if (!authCert) {
throw new Error('用户未注册');
}
@@ -32,12 +33,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('密码错误');
}
const token = generateToken(user._id);
const userDetail = await getUserDetail({ tmbId, userId: user._id });
const token = createJWT(userDetail);
setCookie(res, token);
jsonRes(res, {
data: {
user,
user: userDetail,
token
}
});

4
projects/app/src/pages/api/user/account/loginout.ts
View File

@@ -1,7 +1,7 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { clearCookie } from '@fastgpt/service/support/user/auth';
import { jsonRes } from '@fastgpt/service/common/response';
import { clearCookie } from '@fastgpt/service/support/permission/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {

17
projects/app/src/pages/api/user/account/tokenLogin.ts
View File

@@ -1,24 +1,17 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { connectToDatabase } from '@/service/mongo';
import { getUserDetail } from '@/service/support/user/controller';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
// 根据 id 获取用户信息
const user = await MongoUser.findById(userId);
if (!user) {
throw new Error('账号异常');
}
const { userId, tmbId } = await authCert({ req, authToken: true });
jsonRes(res, {
data: user
data: await getUserDetail({ tmbId, userId })
});
} catch (err) {
jsonRes(res, {

6
projects/app/src/pages/api/user/account/update.ts
View File

@@ -1,8 +1,8 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { jsonRes } from '@fastgpt/service/common/response';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { UserUpdateParams } from '@/types/user';
import { getAIApi, openaiBaseUrl } from '@fastgpt/service/core/ai/config';
import { connectToDatabase } from '@/service/mongo';
@@ -13,7 +13,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
await connectToDatabase();
const { avatar, timezone, openaiAccount } = req.body as UserUpdateParams;
const { userId } = await authUser({ req, authToken: true });
const { userId } = await authCert({ req, authToken: true });
// auth key
if (openaiAccount?.key) {

6
projects/app/src/pages/api/user/account/updatePasswordByOld.ts
View File

@@ -1,7 +1,7 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { connectToDatabase } from '@/service/mongo';
@@ -14,7 +14,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('Params is missing');
}
const { userId } = await authUser({ req, authToken: true });
const { userId } = await authCert({ req, authToken: true });
// auth old password
const user = await MongoUser.findOne({

57
projects/app/src/pages/api/user/getBill.ts
View File

@@ -1,57 +0,0 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { Bill, connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { adaptBill } from '@/utils/adapt';
import { addDays } from 'date-fns';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const {
pageNum = 1,
pageSize = 10,
dateStart = addDays(new Date(), -7),
dateEnd = new Date()
} = req.body as {
pageNum: number;
pageSize: number;
dateStart: Date;
dateEnd: Date;
};
const { userId } = await authUser({ req, authToken: true });
const where = {
userId,
time: {
$gte: dateStart,
$lte: dateEnd
}
};
// get bill record and total by record
const [bills, total] = await Promise.all([
Bill.find(where)
.sort({ time: -1 }) // 按照创建时间倒序排列
.skip((pageNum - 1) * pageSize)
.limit(pageSize),
Bill.countDocuments(where)
]);
jsonRes(res, {
data: {
pageNum,
pageSize,
data: bills.map(adaptBill),
total
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

29
projects/app/src/pages/api/user/getPayOrders.ts
View File

@@ -1,29 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { connectToDatabase } from '@/service/mongo';
import { MongoPay } from '@fastgpt/service/support/wallet/pay/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const records = await MongoPay.find({
userId,
status: { $ne: 'CLOSED' }
})
.sort({ createTime: -1 })
.limit(100);
jsonRes(res, {
data: records
});
} catch (err) {
console.log(err);
jsonRes(res, {
code: 500,
error: err
});
}
}

31
projects/app/src/pages/api/user/inform/countUnread.ts
View File

@@ -1,31 +0,0 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoUserInform } from '@fastgpt/service/support/user/inform/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
if (!req.headers.cookie) {
return jsonRes(res, {
data: 0
});
}
const { userId } = await authUser({ req, authToken: true });
const data = await MongoUserInform.countDocuments({
userId,
read: false
});
jsonRes(res, {
data
});
} catch (err) {
jsonRes(res, {
data: 0
});
}
}

40
projects/app/src/pages/api/user/inform/list.ts
View File

@@ -1,40 +0,0 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { MongoUserInform } from '@fastgpt/service/support/user/inform/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { pageNum, pageSize = 10 } = req.body as {
pageNum: number;
pageSize: number;
};
const [informs, total] = await Promise.all([
MongoUserInform.find({ userId })
.sort({ time: -1 }) // 按照创建时间倒序排列
.skip((pageNum - 1) * pageSize)
.limit(pageSize),
MongoUserInform.countDocuments({ userId })
]);
jsonRes(res, {
data: {
pageNum,
pageSize,
data: informs,
total
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

29
projects/app/src/pages/api/user/inform/read.ts
View File

@@ -1,29 +0,0 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { MongoUserInform } from '@fastgpt/service/support/user/inform/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { id } = req.query as { id: string };
await MongoUserInform.findOneAndUpdate(
{
_id: id,
userId
},
{
read: true
}
);
jsonRes(res);
} catch (err) {
jsonRes(res);
}
}

55
projects/app/src/pages/api/user/inform/send.ts
View File

@@ -1,55 +0,0 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { startSendInform } from '@/service/events/sendInform';
import { MongoUserInform } from '@fastgpt/service/support/user/inform/schema';
import { InformTypeEnum } from '@fastgpt/global/support/user/constant';
import {
sendInform2AllUser,
sendInform2OneUser
} from '@fastgpt/service/support/user/inform/controller';
export type Props = {
type: `${InformTypeEnum}`;
title: string;
content: string;
userId?: string;
};
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authUser({ req, authRoot: true });
jsonRes(res, {
data: await sendInform(req.body),
message: '发送通知成功'
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
export async function sendInform({ type, title, content, userId }: Props) {
if (!type || !title || !content) {
return;
}
try {
if (userId) {
global.sendInformQueue.push(async () => sendInform2OneUser({ type, title, content, userId }));
startSendInform();
return;
}
// send to all user
sendInform2AllUser({ type, title, content });
} catch (error) {
console.log('send inform error', error);
}
}

53
projects/app/src/pages/api/user/promotion/getPromotionData.ts
View File

@@ -1,53 +0,0 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoPromotionRecord } from '@fastgpt/service/support/activity/promotion/schema';
import { authUser } from '@fastgpt/service/support/user/auth';
import mongoose from '@fastgpt/service/common/mongo';
import { MongoUser } from '@fastgpt/service/support/user/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const invitedAmount = await MongoUser.countDocuments({
inviterId: userId
});
// 计算累计合
const countHistory: { totalAmount: number }[] = await MongoPromotionRecord.aggregate([
{
$match: {
userId: new mongoose.Types.ObjectId(userId),
amount: { $gt: 0 }
}
},
{
$group: {
_id: null, // 分组条件，这里使用 null 表示不分组
totalAmount: { $sum: '$amount' } // 计算 amount 字段的总和
}
},
{
$project: {
_id: false, // 排除 _id 字段
totalAmount: true // 只返回 totalAmount 字段
}
}
]);
jsonRes(res, {
data: {
invitedAmount,
earningsAmount: countHistory[0]?.totalAmount || 0
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

43
projects/app/src/pages/api/user/promotion/getPromotions.ts
View File

@@ -1,43 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { connectToDatabase } from '@/service/mongo';
import { authUser } from '@fastgpt/service/support/user/auth';
import { MongoPromotionRecord } from '@fastgpt/service/support/activity/promotion/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
let { pageNum = 1, pageSize = 10 } = req.body as {
pageNum: number;
pageSize: number;
};
const { userId } = await authUser({ req, authToken: true });
const data = await MongoPromotionRecord.find(
{
userId
},
'_id createTime type amount'
)
.sort({ _id: -1 })
.skip((pageNum - 1) * pageSize)
.limit(pageSize);
jsonRes(res, {
data: {
pageNum,
pageSize,
data,
total: await MongoPromotionRecord.countDocuments({
userId
})
}
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

465
projects/app/src/pages/api/v1/chat/completions.ts
View File

@@ -1,49 +1,30 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { authApp } from '@/service/utils/auth';
import { authUser, AuthUserTypeEnum } from '@fastgpt/service/support/user/auth';
import { sseErrRes, jsonRes } from '@/service/response';
import { addLog } from '@/service/utils/tools';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { sseErrRes, jsonRes } from '@fastgpt/service/common/response';
import { addLog } from '@fastgpt/service/common/mongo/controller';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { ChatRoleEnum, ChatSourceEnum, sseResponseEventEnum } from '@/constants/chat';
import {
dispatchHistory,
dispatchChatInput,
dispatchChatCompletion,
dispatchDatasetSearch,
dispatchAnswer,
dispatchClassifyQuestion,
dispatchContentExtract,
dispatchHttpRequest,
dispatchAppRequest,
dispatchRunPlugin,
dispatchPluginInput,
dispatchPluginOutput
} from '@/service/moduleDispatch';
import type { CreateChatCompletionRequest } from '@fastgpt/global/core/ai/type.d';
import type { MessageItemType } from '@/types/core/chat/type';
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
import { sseResponseEventEnum } from '@fastgpt/service/common/response/constant';
import { dispatchModules } from '@/service/moduleDispatch';
import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d';
import type { ChatMessageItemType } from '@fastgpt/global/core/ai/type.d';
import { gptMessage2ChatType, textAdaptGptResponse } from '@/utils/adapt';
import { getChatHistory } from './getHistory';
import { saveChat } from '@/service/utils/chat/saveChat';
import { responseWrite } from '@fastgpt/service/common/response';
import { TaskResponseKeyEnum } from '@/constants/chat';
import { initModuleType } from '@/constants/flow';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { RunningModuleItemType } from '@/types/app';
import type { ModuleItemType } from '@fastgpt/global/core/module/type';
import { pushChatBill } from '@/service/common/bill/push';
import { BillSourceEnum } from '@/constants/user';
import { ChatHistoryItemResType } from '@/types/chat';
import type { UserModelSchema } from '@fastgpt/global/support/user/type';
import { SystemInputEnum, SystemOutputEnum } from '@/constants/app';
import { getSystemTime } from '@fastgpt/global/common/time/timezone';
import { authOutLinkChat } from '@fastgpt/service/support/outLink/auth';
import { pushChatBill } from '@/service/support/wallet/bill/push';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { authOutLinkChat } from '@/service/support/permission/auth/outLink';
import { pushResult2Remote, updateOutLinkUsage } from '@fastgpt/service/support/outLink/tools';
import requestIp from 'request-ip';
import { replaceVariable } from '@/global/common/string/tools';
import type { ModuleDispatchProps } from '@/types/core/chat/type';
import { selectShareResponse } from '@/utils/service/core/chat';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
import { connectToDatabase } from '@/service/mongo';
import { getUserAndAuthBalance } from '@/service/support/permission/auth/user';
import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
import { MongoApp } from '@fastgpt/service/core/app/schema';
type FastGptWebChatProps = {
chatId?: string; // undefined: nonuse history, '': new chat, 'xxxxx': use history
@@ -53,10 +34,10 @@ type FastGptShareChatProps = {
shareId?: string;
authToken?: string;
};
export type Props = CreateChatCompletionRequest &
export type Props = ChatCompletionCreateParams &
FastGptWebChatProps &
FastGptShareChatProps & {
messages: MessageItemType[];
messages: ChatMessageItemType[];
stream?: boolean;
detail?: boolean;
variables: Record<string, any>;
@@ -101,68 +82,98 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
let startTime = Date.now();
/* user auth */
const {
responseDetail: shareResponseDetail,
user,
userId,
appId: authAppid,
authType,
apikey
} = await (async (): Promise<{
user?: UserModelSchema;
responseDetail?: boolean;
userId: string;
appId: string;
authType: `${AuthUserTypeEnum}`;
apikey?: string;
}> => {
const chatMessages = gptMessage2ChatType(messages);
if (chatMessages[chatMessages.length - 1].obj !== ChatRoleEnum.Human) {
chatMessages.pop();
}
// user question
const question = chatMessages.pop();
if (!question) {
throw new Error('Question is empty');
}
/* auth app permission */
const { user, app, responseDetail, authType, apikey, canWrite } = await (async () => {
if (shareId) {
return authOutLinkChat({
const { user, app, authType, responseDetail } = await authOutLinkChat({
shareId,
ip: requestIp.getClientIp(req),
authToken,
question:
(messages[messages.length - 2]?.role === 'user'
? messages[messages.length - 2].content
: messages[messages.length - 1]?.content) || ''
question: question.value
});
return {
user,
app,
responseDetail,
apikey: '',
authType,
canWrite: false
};
}
return authUser({ req, authToken: true, authApiKey: true, authBalance: true });
const {
appId: apiKeyAppId,
tmbId,
authType,
apikey
} = await authCert({
req,
authToken: true,
authApiKey: true
});
const user = await getUserAndAuthBalance({
tmbId,
minBalance: 0
});
// openapi key
if (authType === AuthUserTypeEnum.apikey) {
const app = await MongoApp.findById(apiKeyAppId);
if (!app) {
return Promise.reject('app is empty');
}
return {
user,
app,
responseDetail: detail,
apikey,
authType,
canWrite: false
};
}
if (!appId) {
return Promise.reject('appId is empty');
}
// token
const { app, canWrite } = await authApp({
req,
authToken: true,
appId,
per: 'r'
});
return {
user,
app,
responseDetail: detail,
apikey,
authType,
canWrite: canWrite || false
};
})();
if (!user) {
throw new Error('Account is error');
}
// must have a app
appId = appId ? appId : authAppid;
if (!appId) {
throw new Error('appId is empty');
}
// auth app, get history
const [{ app }, { history }] = await Promise.all([
authApp({
appId,
userId
}),
getChatHistory({ chatId, appId, userId })
]);
const { history } = await getChatHistory({ chatId, tmbId: user.team.tmbId });
const isOwner = !shareId && userId === String(app.userId);
const responseDetail = isOwner || shareResponseDetail;
const isOwner = !shareId && String(user.team.tmbId) === String(app.tmbId);
/* format prompts */
const prompts = history.concat(gptMessage2ChatType(messages));
if (prompts[prompts.length - 1]?.obj === 'AI') {
prompts.pop();
}
// user question
const prompt = prompts.pop();
if (!prompt) {
throw new Error('Question is empty');
}
// set sse response headers
if (stream) {
@@ -175,12 +186,15 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
/* start flow controller */
const { responseData, answerText } = await dispatchModules({
res,
chatId,
modules: app.modules,
user,
teamId: user.team.teamId,
tmbId: user.team.tmbId,
variables,
params: {
history: prompts,
userChatInput: prompt.value
userChatInput: question.value
},
stream,
detail
@@ -190,8 +204,9 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
if (chatId) {
await saveChat({
chatId,
appId,
userId,
appId: app._id,
teamId: user.team.teamId,
tmbId: user.team.tmbId,
variables,
isOwner, // owner update use time
shareId,
@@ -205,7 +220,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
return ChatSourceEnum.online;
})(),
content: [
prompt,
question,
{
dataId: messages[messages.length - 1].dataId,
obj: ChatRoleEnum.AI,
@@ -219,7 +234,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
addLog.info(`completions running time: ${(Date.now() - startTime) / 1000}s`);
/* select fe response field */
const feResponseData = isOwner ? responseData : selectShareResponse({ responseData });
const feResponseData = canWrite ? responseData : selectShareResponse({ responseData });
if (stream) {
responseWrite({
@@ -264,8 +279,9 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// add record
const { total } = pushChatBill({
appName: app.name,
appId,
userId,
appId: app._id,
teamId: user.team.teamId,
tmbId: user.team.tmbId,
source: (() => {
if (authType === 'apikey') return BillSourceEnum.api;
if (shareId) return BillSourceEnum.shareLink;
@@ -281,11 +297,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
total
});
}
!!apikey &&
if (apikey) {
updateApiKeyUsage({
apikey,
usage: total
});
}
} catch (err: any) {
if (stream) {
sseErrRes(res, err);
@@ -299,266 +316,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
}
});
/* running */
export async function dispatchModules({
res,
modules,
user,
params = {},
variables = {},
stream = false,
detail = false
}: {
res: NextApiResponse;
modules: ModuleItemType[];
user: UserModelSchema;
params?: Record<string, any>;
variables?: Record<string, any>;
stream?: boolean;
detail?: boolean;
}) {
variables = {
...getSystemVariable({ timezone: user.timezone }),
...variables
};
const runningModules = loadModules(modules, variables);
// let storeData: Record<string, any> = {}; // after module used
let chatResponse: ChatHistoryItemResType[] = []; // response request and save to database
let chatAnswerText = ''; // AI answer
let runningTime = Date.now();
function pushStore(
{ inputs = [] }: RunningModuleItemType,
{
answerText = '',
responseData
}: {
answerText?: string;
responseData?: ChatHistoryItemResType | ChatHistoryItemResType[];
}
) {
const time = Date.now();
if (responseData) {
if (Array.isArray(responseData)) {
chatResponse = chatResponse.concat(responseData);
} else {
chatResponse.push({
...responseData,
runningTime: +((time - runningTime) / 1000).toFixed(2)
});
}
}
runningTime = time;
const isResponseAnswerText =
inputs.find((item) => item.key === SystemInputEnum.isResponseAnswerText)?.value ?? true;
if (isResponseAnswerText) {
chatAnswerText += answerText;
}
}
function moduleInput(
module: RunningModuleItemType,
data: Record<string, any> = {}
): Promise<any> {
const checkInputFinish = () => {
return !module.inputs.find((item: any) => item.value === undefined);
};
const updateInputValue = (key: string, value: any) => {
const index = module.inputs.findIndex((item: any) => item.key === key);
if (index === -1) return;
module.inputs[index].value = value;
};
const set = new Set();
return Promise.all(
Object.entries(data).map(([key, val]: any) => {
updateInputValue(key, val);
if (!set.has(module.moduleId) && checkInputFinish()) {
set.add(module.moduleId);
// remove switch
updateInputValue(SystemInputEnum.switch, undefined);
return moduleRun(module);
}
})
);
}
function moduleOutput(
module: RunningModuleItemType,
result: Record<string, any> = {}
): Promise<any> {
pushStore(module, result);
return Promise.all(
module.outputs.map((outputItem) => {
if (result[outputItem.key] === undefined) return;
/* update output value */
outputItem.value = result[outputItem.key];
/* update target */
return Promise.all(
outputItem.targets.map((target: any) => {
// find module
const targetModule = runningModules.find((item) => item.moduleId === target.moduleId);
if (!targetModule) return;
return moduleInput(targetModule, { [target.key]: outputItem.value });
})
);
})
);
}
async function moduleRun(module: RunningModuleItemType): Promise<any> {
if (res.closed) return Promise.resolve();
if (stream && detail && module.showStatus) {
responseStatus({
res,
name: module.name,
status: 'running'
});
}
// get fetch params
const params: Record<string, any> = {};
module.inputs.forEach((item: any) => {
params[item.key] = item.value;
});
const props: ModuleDispatchProps<Record<string, any>> = {
res,
stream,
detail,
variables,
outputs: module.outputs,
user,
inputs: params
};
const dispatchRes: Record<string, any> = await (async () => {
const callbackMap: Record<string, Function> = {
[FlowNodeTypeEnum.historyNode]: dispatchHistory,
[FlowNodeTypeEnum.questionInput]: dispatchChatInput,
[FlowNodeTypeEnum.answerNode]: dispatchAnswer,
[FlowNodeTypeEnum.chatNode]: dispatchChatCompletion,
[FlowNodeTypeEnum.datasetSearchNode]: dispatchDatasetSearch,
[FlowNodeTypeEnum.classifyQuestion]: dispatchClassifyQuestion,
[FlowNodeTypeEnum.contentExtract]: dispatchContentExtract,
[FlowNodeTypeEnum.httpRequest]: dispatchHttpRequest,
[FlowNodeTypeEnum.runApp]: dispatchAppRequest,
[FlowNodeTypeEnum.pluginModule]: dispatchRunPlugin,
[FlowNodeTypeEnum.pluginInput]: dispatchPluginInput,
[FlowNodeTypeEnum.pluginOutput]: dispatchPluginOutput
};
if (callbackMap[module.flowType]) {
return callbackMap[module.flowType](props);
}
return {};
})();
const formatResponseData = (() => {
if (!dispatchRes[TaskResponseKeyEnum.responseData]) return undefined;
if (Array.isArray(dispatchRes[TaskResponseKeyEnum.responseData]))
return dispatchRes[TaskResponseKeyEnum.responseData];
return {
...dispatchRes[TaskResponseKeyEnum.responseData],
moduleName: module.name,
moduleType: module.flowType
};
})();
return moduleOutput(module, {
[SystemOutputEnum.finish]: true,
...dispatchRes,
[TaskResponseKeyEnum.responseData]: formatResponseData
});
}
// start process width initInput
const initModules = runningModules.filter((item) => initModuleType[item.flowType]);
await Promise.all(initModules.map((module) => moduleInput(module, params)));
return {
[TaskResponseKeyEnum.answerText]: chatAnswerText,
[TaskResponseKeyEnum.responseData]: chatResponse
};
}
/* init store modules to running modules */
function loadModules(
modules: ModuleItemType[],
variables: Record<string, any>
): RunningModuleItemType[] {
return modules.map((module) => {
return {
moduleId: module.moduleId,
name: module.name,
flowType: module.flowType,
showStatus: module.showStatus,
inputs: module.inputs
.filter((item) => item.connected) // filter unconnected target input
.map((item) => {
if (typeof item.value !== 'string') {
return {
key: item.key,
value: item.value
};
}
// variables replace
const replacedVal = replaceVariable(item.value, variables);
return {
key: item.key,
value: replacedVal
};
}),
outputs: module.outputs
.map((item) => ({
key: item.key,
answer: item.key === TaskResponseKeyEnum.answerText,
value: undefined,
targets: item.targets
}))
.sort((a, b) => {
// finish output always at last
if (a.key === SystemOutputEnum.finish) return 1;
if (b.key === SystemOutputEnum.finish) return -1;
return 0;
})
};
});
}
/* sse response modules staus */
export function responseStatus({
res,
status,
name
}: {
res: NextApiResponse;
status?: 'running' | 'finish';
name?: string;
}) {
if (!name) return;
responseWrite({
res,
event: sseResponseEventEnum.moduleStatus,
data: JSON.stringify({
status: 'running',
name
})
});
}
/* get system variable */
export function getSystemVariable({ timezone }: { timezone: string }) {
return {
cTime: getSystemTime(timezone)
};
}
export const config = {
api: {
responseLimit: '20mb'

25
projects/app/src/pages/api/v1/chat/getHistory.ts
View File

@@ -1,10 +1,11 @@
// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@/service/response';
import { authUser } from '@fastgpt/service/support/user/auth';
import { ChatItem, connectToDatabase } from '@/service/mongo';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { connectToDatabase } from '@/service/mongo';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { Types } from '@fastgpt/service/common/mongo';
import type { ChatItemType } from '@/types/chat';
import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
export type Props = {
appId?: string;
@@ -16,13 +17,13 @@ export type Response = { history: ChatItemType[] };
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { userId } = await authUser({ req, authToken: true });
const { tmbId } = await authCert({ req, authToken: true });
const { chatId, limit } = req.body as Props;
jsonRes<Response>(res, {
data: await getChatHistory({
chatId,
userId,
tmbId,
limit
})
});
@@ -36,20 +37,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
export async function getChatHistory({
chatId,
userId,
appId,
tmbId,
limit = 30
}: Props & { userId: string }): Promise<Response> {
if (!chatId || !appId) {
}: Props & { tmbId: string }): Promise<Response> {
if (!chatId) {
return { history: [] };
}
const history = await ChatItem.aggregate([
const history = await MongoChatItem.aggregate([
{
$match: {
chatId,
appId: new Types.ObjectId(appId),
userId: new Types.ObjectId(userId)
tmbId: new Types.ObjectId(tmbId)
}
},
{

58
projects/app/src/pages/api/v1/embeddings.ts Normal file
View File

@@ -0,0 +1,58 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
import { connectToDatabase } from '@/service/mongo';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { getVectorsByText, GetVectorProps } from '@/service/core/ai/vector';
type Props = GetVectorProps & {
billId?: string;
};
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let { input, model, billId } = req.body as Props;
await connectToDatabase();
const { teamId, tmbId } = await authCert({ req, authToken: true });
if (!Array.isArray(input) || typeof input !== 'string') {
throw new Error('input is nor array or string');
}
await authTeamBalance(teamId);
const { tokenLen, vectors } = await getVectorsByText({ input, model });
pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
model,
billId
});
jsonRes(res, {
data: {
object: 'list',
data: vectors.map((item, index) => ({
object: 'embedding',
index: index,
embedding: item
})),
model,
usage: {
prompt_tokens: tokenLen,
total_tokens: tokenLen
}
}
});
} catch (err) {
console.log(err);
jsonRes(res, {
code: 500,
error: err
});
}
});