gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

4.6.7-alpha commit (#743)

Browse Source 
Co-authored-by: Archer <545436317@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-01-19 11:17:28 +08:00
committed by GitHub
parent 8ee7407c4c
commit c031e6dcc9
324 changed files with 8509 additions and 4757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
projects/app/src/pages/api/admin/initv46-2.ts
View File

@@ -1,184 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { defaultQAModels } from '@fastgpt/global/core/ai/model';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
try {
await Promise.allSettled([
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN data_id VARCHAR(50);`),
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN q DROP NOT NULL;`), // q can null
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN a DROP NOT NULL;`), // a can null
PgClient.query(
`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id TYPE VARCHAR(50) USING team_id::VARCHAR(50);`
), // team_id varchar
PgClient.query(
`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id TYPE VARCHAR(50) USING tmb_id::VARCHAR(50);`
), // tmb_id varchar
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id SET NOT NULL;`), // team_id not null
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id SET NOT NULL;`), // tmb_id not null
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN dataset_id SET NOT NULL;`), // dataset_id not null
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN collection_id SET NOT NULL;`) // collection_id not null
]);
} catch (error) {}
try {
await initPgData();
} catch (error) {}
await MongoDataset.updateMany(
{},
{
agentModel: defaultQAModels[0].model
}
);
jsonRes(res, {
data: await init(limit),
message:
'初始化任务已开始，请注意日志进度。可通过 select count(id) from modeldata where data_id is null; 检查是否完全初始化，如果结果为 0 ，则完全初始化。'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
type PgItemType = {
id: string;
q: string;
a: string;
dataset_id: string;
collection_id: string;
team_id: string;
tmb_id: string;
};
async function initPgData() {
const limit = 10;
const { rows } = await PgClient.query<{ user_id: string }>(`
SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id='null';
`);
console.log('init pg', rows.length);
let success = 0;
for (let i = 0; i < limit; i++) {
init(i);
}
async function init(index: number): Promise<any> {
const userId = rows[index]?.user_id;
if (!userId) return;
try {
const tmb = await getUserDefaultTeam({ userId });
console.log(tmb);
// update pg
await PgClient.query(
`Update ${PgDatasetTableName} set team_id = '${String(tmb.teamId)}', tmb_id = '${String(
tmb.tmbId
)}' where user_id = '${userId}' AND team_id='null';`
);
console.log(++success);
init(index + limit);
} catch (error) {
if (error === 'default team not exist') {
return;
}
console.log(error);
await delay(1000);
return init(index);
}
}
}
async function init(limit: number): Promise<any> {
const { rows: idList } = await PgClient.query<{ id: string }>(
`SELECT id FROM ${PgDatasetTableName} WHERE data_id IS NULL`
);
console.log('totalCount', idList.length);
if (idList.length === 0) return;
for (let i = 0; i < limit; i++) {
initData(i);
}
async function initData(index: number): Promise<any> {
const dataId = idList[index]?.id;
if (!dataId) {
console.log('done');
return;
}
// get limit data where data_id is null
const { rows } = await PgClient.query<PgItemType>(
`SELECT id,q,a,dataset_id,collection_id,team_id,tmb_id FROM ${PgDatasetTableName} WHERE id=${dataId};`
);
const data = rows[0];
if (!data) {
console.log('done');
return;
}
let id = '';
try {
// create mongo data and update data_id
const { _id } = await MongoDatasetData.create({
teamId: data.team_id.trim(),
tmbId: data.tmb_id.trim(),
datasetId: data.dataset_id,
collectionId: data.collection_id,
q: data.q,
a: data.a,
fullTextToken: '',
indexes: [
{
defaultIndex: !data.a,
type: data.a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk,
dataId: data.id,
text: data.q
}
]
});
id = _id;
// update pg data_id
await PgClient.query(
`UPDATE ${PgDatasetTableName} SET data_id='${String(_id)}' WHERE id=${dataId};`
);
console.log(++success);
return initData(index + limit);
} catch (error) {
console.log(error);
console.log(data);
try {
if (id) {
await MongoDatasetData.findByIdAndDelete(id);
}
} catch (error) {}
await delay(500);
return initData(index);
}
}
}

173
projects/app/src/pages/api/admin/initv46-fix.ts
View File

@@ -1,173 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { Types, connectionMongo } from '@fastgpt/service/common/mongo';
import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant';
import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller';
import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
await init(limit);
await initCollectionFileTeam(limit);
jsonRes(res, {});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
type PgItemType = {
id: string;
q: string;
a: string;
dataset_id: string;
collection_id: string;
data_id: string;
};
async function init(limit: number): Promise<any> {
const { rows } = await PgClient.query<{ id: string; data_id: string }>(
`SELECT id,data_id FROM ${PgDatasetTableName} WHERE team_id = tmb_id`
);
console.log('totalCount', rows.length);
await delay(2000);
if (rows.length === 0) return;
for (let i = 0; i < limit; i++) {
initData(i);
}
async function initData(index: number): Promise<any> {
const item = rows[index];
if (!item) {
console.log('done');
return;
}
// get mongo
const mongoData = await MongoDatasetData.findById(item.data_id, '_id teamId tmbId');
if (!mongoData) {
return initData(index + limit);
}
try {
// find team owner
const db = connectionMongo?.connection?.db;
const TeamMember = db.collection(TeamMemberCollectionName);
const tmb = await TeamMember.findOne({
teamId: new Types.ObjectId(mongoData.teamId),
role: 'owner'
});
if (!tmb) {
return initData(index + limit);
}
// update mongo and pg tmb_id
await MongoDatasetData.findByIdAndUpdate(item.data_id, {
tmbId: tmb._id
});
await PgClient.query(
`UPDATE ${PgDatasetTableName} SET tmb_id = '${String(tmb._id)}' WHERE id = '${item.id}'`
);
console.log(++success);
return initData(index + limit);
} catch (error) {
console.log(error);
await delay(500);
return initData(index);
}
}
}
async function initCollectionFileTeam(limit: number) {
/* init user default Team */
const DatasetFile = getGFSCollection('dataset');
const matchWhere = {
$or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }]
};
const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([
{
$match: matchWhere
},
{
$group: {
_id: '$metadata.userId', // 按 metadata.userId 分组以去重
userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId
}
},
{
$project: {
_id: 0, // 不显示 _id 字段
userId: 1 // 只显示 userId 字段
}
}
]).toArray();
const users = uniqueUsersWithNoTeamId;
console.log('un init total', users.length);
// limit 组一次
const userArr: any[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const item of userArr) {
await Promise.all(item.map((item) => init(item.userId)));
success += limit;
console.log(success);
}
async function init(userId: string): Promise<any> {
try {
const tmb = await getUserDefaultTeam({
userId
});
await DatasetFile.updateMany(
{
'metadata.userId': String(userId),
...matchWhere
},
{
$set: {
'metadata.teamId': String(tmb.teamId),
'metadata.tmbId': String(tmb.tmbId)
}
}
);
} catch (error) {
if (error === 'team not exist' || error === 'tmbId or userId is required') {
return;
}
console.log(error);
await delay(1000);
return init(userId);
}
}
}

330
projects/app/src/pages/api/admin/initv46.ts
View File

@@ -1,330 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema';
import {
createDefaultTeam,
getUserDefaultTeam
} from '@fastgpt/service/support/user/team/controller';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { UserModelSchema } from '@fastgpt/global/support/user/type';
import { delay } from '@fastgpt/global/common/system/utils';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
import { POST } from '@fastgpt/service/common/api/plusRequest';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50, maxSize = 3 } = req.body as { limit: number; maxSize: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
await initDefaultTeam(limit, maxSize);
await initMongoTeamId(limit);
await initDatasetAndApp();
await initCollectionFileTeam(limit);
if (FastGPTProUrl) {
POST('/admin/init46');
}
await initPgData();
jsonRes(res, {
data: {}
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
async function initDefaultTeam(limit: number, maxSize: number) {
/* init user default Team */
const users = await MongoUser.find({}, '_id balance');
console.log('init user default team', users.length);
// 100 组一次
const userArr: UserModelSchema[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const users of userArr) {
await Promise.all(users.map(init));
success += limit;
console.log(success);
}
async function init(user: UserModelSchema): Promise<any> {
try {
await createDefaultTeam({
userId: user._id,
balance: user.balance,
maxSize
});
} catch (error) {
console.log(error);
await delay(1000);
return init(user);
}
}
}
async function initMongoTeamId(limit: number) {
const mongoSchema = [
{
label: 'MongoPlugin',
schema: MongoPlugin
},
{
label: 'MongoChat',
schema: MongoChat
},
{
label: 'MongoChatItem',
schema: MongoChatItem
},
{
label: 'MongoApp',
schema: MongoApp
},
{
label: 'MongoDataset',
schema: MongoDataset
},
{
label: 'MongoDatasetCollection',
schema: MongoDatasetCollection
},
{
label: 'MongoDatasetTraining',
schema: MongoDatasetTraining
},
{
label: 'MongoBill',
schema: MongoBill
},
{
label: 'MongoOutLink',
schema: MongoOutLink
},
{
label: 'MongoOpenApi',
schema: MongoOpenApi
}
];
/* init user default Team */
for await (const item of mongoSchema) {
console.log('start init', item.label);
await initTeamTmbId(item.schema);
console.log('finish init', item.label);
}
async function initTeamTmbId(schema: any) {
const emptyWhere = {
$or: [{ teamId: { $exists: false } }, { teamId: null }]
};
const uniqueUsersWithNoTeamId = await schema.aggregate([
{
$match: emptyWhere
},
{
$group: {
_id: '$userId', // 按 userId 分组以去重
userId: { $first: '$userId' } // 保留第一个出现的 userId
}
},
{
$project: {
_id: 0, // 不显示 _id 字段
userId: 1 // 只显示 userId 字段
}
}
]);
const users = uniqueUsersWithNoTeamId;
console.log('un init total', users.length);
// limit 组一次
const userArr: any[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const users of userArr) {
await Promise.all(users.map((item) => init(item.userId)));
success += limit;
console.log(success);
}
async function init(userId: string): Promise<any> {
try {
const tmb = await getUserDefaultTeam({ userId });
await schema.updateMany(
{
userId,
...emptyWhere
},
{
teamId: tmb.teamId,
tmbId: tmb.tmbId
}
);
} catch (error) {
if (error === 'team not exist' || error === 'tmbId or userId is required') {
return;
}
console.log(error);
await delay(1000);
return init(userId);
}
}
}
}
async function initDatasetAndApp() {
await MongoDataset.updateMany(
{},
{
$set: {
permission: PermissionTypeEnum.private
}
}
);
await MongoApp.updateMany(
{},
{
$set: {
permission: PermissionTypeEnum.private
}
}
);
}
async function initCollectionFileTeam(limit: number) {
/* init user default Team */
const DatasetFile = getGFSCollection('dataset');
const matchWhere = {
$or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }]
};
const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([
{
$match: matchWhere
},
{
$group: {
_id: '$metadata.userId', // 按 metadata.userId 分组以去重
userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId
}
},
{
$project: {
_id: 0, // 不显示 _id 字段
userId: 1 // 只显示 userId 字段
}
}
]).toArray();
const users = uniqueUsersWithNoTeamId;
console.log('un init total', users.length);
// limit 组一次
const userArr: any[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const item of userArr) {
await Promise.all(item.map((item) => init(item.userId)));
success += limit;
console.log(success);
}
async function init(userId: string): Promise<any> {
try {
const tmb = await getUserDefaultTeam({
userId
});
await DatasetFile.updateMany(
{
'metadata.userId': String(userId),
...matchWhere
},
{
$set: {
'metadata.teamId': String(tmb.teamId),
'metadata.tmbId': String(tmb.tmbId)
}
}
);
} catch (error) {
if (error === 'team not exist' || error === 'tmbId or userId is required') {
return;
}
console.log(error);
await delay(1000);
return init(userId);
}
}
}
async function initPgData() {
const limit = 10;
// add column
try {
await Promise.allSettled([
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id VARCHAR(50);`),
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id VARCHAR(50);`),
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN user_id DROP NOT NULL;`)
]);
} catch (error) {
console.log(error);
console.log('column exists');
}
const { rows } = await PgClient.query<{ user_id: string }>(`
SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id IS NULL;
`);
console.log('init pg', rows.length);
let success = 0;
for (let i = 0; i < limit; i++) {
init(i);
}
async function init(index: number): Promise<any> {
const userId = rows[index]?.user_id;
if (!userId) return;
try {
const tmb = await getUserDefaultTeam({ userId });
// update pg
await PgClient.query(
`Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id IS NULL;`
);
console.log(++success);
init(index + limit);
} catch (error) {
if (error === 'default team not exist') {
return;
}
console.log(error);
await delay(1000);
return init(index);
}
}
}

2
projects/app/src/pages/api/admin/initv463.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
let success = 0;

106
projects/app/src/pages/api/admin/initv467.ts Normal file
View File

@@ -0,0 +1,106 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { MongoImage } from '@fastgpt/service/common/file/image/schema';
import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type';
import { delay } from '@fastgpt/global/common/system/utils';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { getNanoid } from '@fastgpt/global/common/string/tools';
let success = 0;
let deleteImg = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { test = false } = req.body as { test: boolean };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
deleteImg = 0;
// 取消 pg tmb_id 和 data_id 的null
await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id DROP NOT NULL;`);
await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN data_id DROP NOT NULL;`);
// 重新绑定 images 和 collections
const images = await MongoImage.find(
{ 'metadata.fileId': { $exists: true } },
'_id metadata'
).lean();
// 去除 fileId 相同的数据
const fileIdMap = new Map<string, MongoImageSchemaType>();
images.forEach((image) => {
// @ts-ignore
const fileId = image.metadata?.fileId;
if (!fileIdMap.has(fileId) && fileId) {
fileIdMap.set(fileId, image);
}
});
const images2 = Array.from(fileIdMap.values());
console.log('total image list', images2.length);
for await (const image of images2) {
await initImages(image, test);
}
jsonRes(res, {
data: success,
message: 'success'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
export const initImages = async (image: MongoImageSchemaType, test: boolean): Promise<any> => {
try {
//@ts-ignore
const fileId = image.metadata.fileId as string;
if (!fileId) return;
// 找到集合
const collection = await MongoDatasetCollection.findOne({ fileId }, '_id metadata').lean();
if (!collection) {
deleteImg++;
console.log('deleteImg', deleteImg);
if (test) return;
return MongoImage.deleteOne({ _id: image._id });
}
const relatedImageId = getNanoid(24);
// update image
if (!test) {
await Promise.all([
MongoImage.updateMany(
{ 'metadata.fileId': fileId },
{ $set: { 'metadata.relatedId': relatedImageId } }
),
MongoDatasetCollection.findByIdAndUpdate(collection._id, {
$set: {
'metadata.relatedImgId': relatedImageId
}
})
]);
}
success++;
console.log('success', success);
} catch (error) {
console.log(error);
await delay(1000);
return initImages(image, test);
}
};

95
projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts
View File

@@ -1,95 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import {
delFileByFileIdList,
getGFSCollection
} from '@fastgpt/service/common/file/gridfs/controller';
import { addLog } from '@fastgpt/service/common/system/log';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { delay } from '@fastgpt/global/common/system/utils';
/*
check dataset.files data. If there is no match in dataset.collections, delete it
*/
let deleteFileAmount = 0;
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const {
startDay = 10,
endDay = 3,
limit = 30
} = req.body as { startDay?: number; endDay?: number; limit?: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
// start: now - maxDay, end: now - 3 day
const start = new Date(Date.now() - startDay * 24 * 60 * 60 * 1000);
const end = new Date(Date.now() - endDay * 24 * 60 * 60 * 1000);
deleteFileAmount = 0;
checkFiles(start, end, limit);
jsonRes(res, {
message: 'success'
});
} catch (error) {
addLog.error(`check valid dataset files error`, error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function checkFiles(start: Date, end: Date, limit: number) {
const collection = getGFSCollection('dataset');
const where = {
uploadDate: { $gte: start, $lte: end }
};
// 1. get all _id
const ids = await collection
.find(where, {
projection: {
_id: 1
}
})
.toArray();
console.log('total files', ids.length);
for (let i = 0; i < limit; i++) {
check(i);
}
async function check(index: number): Promise<any> {
const id = ids[index];
if (!id) {
console.log(`检测完成，共删除 ${deleteFileAmount} 个无效文件`);
return;
}
try {
const { _id } = id;
// 2. find fileId in dataset.collections
const hasCollection = await MongoDatasetCollection.countDocuments({ fileId: _id });
// 3. if not found, delete file
if (hasCollection === 0) {
await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(_id)] });
console.log('delete file', _id);
deleteFileAmount++;
}
index % 100 === 0 && console.log(index);
return check(index + limit);
} catch (error) {
console.log(error);
await delay(2000);
return check(index);
}
}
}

30
projects/app/src/pages/api/common/file/upload.ts
View File

@@ -18,32 +18,24 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
try {
const { userId, teamId, tmbId } = await authCert({ req, authToken: true });
const { files, bucketName, metadata } = await upload.doUpload(req, res);
filePaths = files.map((file) => file.path);
const { file, bucketName, metadata } = await upload.doUpload(req, res);
filePaths = [file.path];
await connectToDatabase();
if (!bucketName) {
throw new Error('bucketName is empty');
}
const upLoadResults = await Promise.all(
files.map((file) =>
uploadFile({
teamId,
tmbId,
bucketName,
path: file.path,
filename: file.originalname,
metadata: {
...metadata,
contentType: file.mimetype,
userId
}
})
)
);
const upLoadResults = await uploadFile({
teamId,
tmbId,
bucketName,
path: file.path,
filename: file.originalname,
contentType: file.mimetype,
metadata: metadata
});
jsonRes(res, {
data: upLoadResults

3
projects/app/src/pages/api/common/system/getInitData.ts
View File

@@ -55,7 +55,8 @@ const defaultFeConfigs: FastGPTFeConfigsType = {
websiteSyncLimitMinuted: 0
},
scripts: [],
favicon: '/favicon.ico'
favicon: '/favicon.ico',
uploadFileMaxSize: 500
};
export async function getInitConfig() {

40
projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
View File

@@ -7,7 +7,7 @@ import { jsonRes } from '@fastgpt/service/common/response';
import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d';
import type { ModuleItemType } from '@fastgpt/global/core/module/type';
import { FormatForm2ModulesProps } from '@fastgpt/global/core/app/api';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import { getExtractModel } from '@/service/core/ai/model';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -37,7 +37,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
return [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -49,7 +49,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -58,7 +58,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -93,7 +93,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -189,7 +189,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -268,7 +268,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
const modules: ModuleItemType[] = [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -280,7 +280,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -289,17 +289,13 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
{
moduleId: 'vuc92c',
key: 'userChatInput'
},
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
@@ -307,7 +303,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
},
{
moduleId: 'datasetSearch',
name: '知识库搜索',
name: 'core.module.template.Dataset search',
avatar: '/imgs/module/db.png',
flowType: 'datasetSearchNode',
showStatus: true,
@@ -447,6 +443,18 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
valueType: 'boolean',
type: 'source',
targets: []
},
{
key: 'userChatInput',
label: 'core.module.input.label.user question',
type: 'hidden',
valueType: 'string',
targets: [
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
]
},
@@ -473,7 +481,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -569,7 +577,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:

38
projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts
View File

@@ -33,7 +33,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
return [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -45,7 +45,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -54,7 +54,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -89,7 +89,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -185,7 +185,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -264,7 +264,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
const modules: ModuleItemType[] = [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -276,7 +276,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -285,17 +285,13 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
{
moduleId: 'vuc92c',
key: 'userChatInput'
},
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
@@ -303,7 +299,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
},
{
moduleId: 'datasetSearch',
name: '知识库搜索',
name: 'core.module.template.Dataset search',
avatar: '/imgs/module/db.png',
flowType: 'datasetSearchNode',
showStatus: true,
@@ -457,6 +453,18 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
valueType: 'boolean',
type: 'source',
targets: []
},
{
key: 'userChatInput',
label: 'core.module.input.label.user question',
type: 'hidden',
valueType: 'string',
targets: [
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
]
},
@@ -483,7 +491,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -579,7 +587,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:

19
projects/app/src/pages/api/core/app/getChatLogs.ts
View File

@@ -8,6 +8,7 @@ import { Types } from '@fastgpt/service/common/mongo';
import { addDays } from 'date-fns';
import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -28,8 +29,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { teamId } = await authApp({ req, authToken: true, appId, per: 'w' });
const where = {
appId: new Types.ObjectId(appId),
teamId: new Types.ObjectId(teamId),
appId: new Types.ObjectId(appId),
updateTime: {
$gte: new Date(dateStart),
$lte: new Date(dateEnd)
@@ -41,18 +42,26 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
{ $match: where },
{
$lookup: {
from: 'chatitems',
let: { chat_id: '$chatId' },
from: ChatItemCollectionName,
let: { chatId: '$chatId' },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$chatId', '$$chat_id'] },
{ $eq: ['$appId', new Types.ObjectId(appId)] }
{ $eq: ['$appId', new Types.ObjectId(appId)] },
{ $eq: ['$chatId', '$$chatId'] }
]
}
}
},
{
$project: {
userGoodFeedback: 1,
userBadFeedback: 1,
customFeedbacks: 1,
adminFeedback: 1
}
}
],
as: 'chatitems'

3
projects/app/src/pages/api/core/chat/clearHistories.ts
View File

@@ -35,16 +35,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
return Promise.reject('Param are error');
})();
console.log(match);
// find chatIds
const list = await MongoChat.find(match, 'chatId').lean();
const idList = list.map((item) => item.chatId);
await MongoChatItem.deleteMany({
appId,
chatId: { $in: idList }
});
await MongoChat.deleteMany({
appId,
chatId: { $in: idList }
});

2
projects/app/src/pages/api/core/chat/delHistory.ts
View File

@@ -23,9 +23,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
await MongoChatItem.deleteMany({
appId,
chatId
});
await MongoChat.findOneAndRemove({
appId,
chatId
});

2
projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts
View File

@@ -27,6 +27,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await MongoChatItem.findOneAndUpdate(
{
appId,
chatId,
dataId: chatItemId
},
{

13
projects/app/src/pages/api/core/chat/feedback/closeCustom.ts
View File

@@ -2,14 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import type {
AdminUpdateFeedbackParams,
CloseCustomFeedbackParams
} from '@/global/core/chat/api.d';
import type { CloseCustomFeedbackParams } from '@/global/core/chat/api.d';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { autChatCrud } from '@/service/support/permission/auth/chat';
/* 初始化我的聊天框，需要身份验证 */
/* remove custom feedback */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
@@ -29,13 +26,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await authCert({ req, authToken: true });
await MongoChatItem.findOneAndUpdate(
{ dataId: chatItemId },
{ appId, chatId, dataId: chatItemId },
{ $unset: { [`customFeedbacks.${index}`]: 1 } }
);
await MongoChatItem.findOneAndUpdate(
{ dataId: chatItemId },
{ $pull: { customFeedbacks: null } }
);
jsonRes(res);
} catch (err) {

1
projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts
View File

@@ -29,6 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await MongoChatItem.findOneAndUpdate(
{
appId,
chatId,
dataId: chatItemId
},

2
projects/app/src/pages/api/core/chat/getHistories.ts
View File

@@ -31,8 +31,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
if (appId) {
const { tmbId } = await authCert({ req, authToken: true });
return {
appId,
tmbId,
appId,
source: ChatSourceEnum.online
};
}

3
projects/app/src/pages/api/core/chat/init.ts
View File

@@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
appId,
per: 'r'
}),
chatId ? MongoChat.findOne({ chatId }) : undefined
chatId ? MongoChat.findOne({ appId, chatId }) : undefined
]);
// auth chat permission
@@ -41,6 +41,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// get app and history
const { history } = await getChatItems({
appId,
chatId,
limit: 30,
field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${

5
projects/app/src/pages/api/core/chat/item/delete.ts
View File

@@ -25,8 +25,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
await MongoChatItem.deleteOne({
dataId: contentId,
chatId
appId,
chatId,
dataId: contentId
});
jsonRes(res);

2
projects/app/src/pages/api/core/chat/item/getSpeech.ts
View File

@@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
try {
pushAudioSpeechBill({
model: model,
textLen: input.length,
charsLength: input.length,
tmbId,
teamId,
source: authType2BillSource({ authType })

3
projects/app/src/pages/api/core/chat/outLink/init.ts
View File

@@ -26,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// auth app permission
const [tmb, chat, app] = await Promise.all([
MongoTeamMember.findById(shareChat.tmbId, '_id userId').populate('userId', 'avatar').lean(),
MongoChat.findOne({ chatId, shareId }).lean(),
MongoChat.findOne({ appId, chatId, shareId }).lean(),
MongoApp.findById(appId).lean()
]);
@@ -40,6 +40,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
const { history } = await getChatItems({
appId: app._id,
chatId,
limit: 30,
field: `dataId obj value userGoodFeedback userBadFeedback ${

2
projects/app/src/pages/api/core/chat/updateHistory.ts
View File

@@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
await MongoChat.findOneAndUpdate(
{ chatId },
{ appId, chatId },
{
...(customTitle !== undefined && { customTitle }),
...(top !== undefined && { top })

2
projects/app/src/pages/api/core/dataset/allDataset.ts
View File

@@ -6,7 +6,7 @@ import { getVectorModel } from '@/service/core/ai/model';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
/* get all dataset by teamId or tmbId */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

85
projects/app/src/pages/api/core/dataset/collection/create/file.ts Normal file
View File

@@ -0,0 +1,85 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller';
import { getUploadModel } from '@fastgpt/service/common/file/multer';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { FileCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
/**
* Creates the multer uploader
*/
const upload = getUploadModel({
maxSize: 500 * 1024 * 1024
});
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let filePaths: string[] = [];
const { datasetId } = req.query as { datasetId: string };
try {
await connectToDatabase();
const { teamId, tmbId } = await authDataset({
req,
authToken: true,
authApiKey: true,
per: 'w',
datasetId
});
const { file, bucketName, data } = await upload.doUpload<FileCreateDatasetCollectionParams>(
req,
res
);
filePaths = [file.path];
if (!file || !bucketName) {
throw new Error('file is empty');
}
const { fileMetadata, collectionMetadata, ...collectionData } = data;
// upload file and create collection
const fileId = await uploadFile({
teamId,
tmbId,
bucketName,
path: file.path,
filename: file.originalname,
contentType: file.mimetype,
metadata: fileMetadata
});
// create collection
const collectionId = await createOneCollection({
...collectionData,
metadata: collectionMetadata,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.file,
fileId
});
jsonRes(res, {
data: collectionId
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
removeFilesByPaths(filePaths);
}
export const config = {
api: {
bodyParser: false
}
};

5
projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts → projects/app/src/pages/api/core/dataset/collection/create/link.ts
View File

@@ -7,7 +7,10 @@ import { connectToDatabase } from '@/service/mongo';
import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';

12
projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts → projects/app/src/pages/api/core/dataset/collection/create/text.ts
View File

@@ -7,11 +7,14 @@ import { connectToDatabase } from '@/service/mongo';
import type { TextCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller';
import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller';
import { hashStr } from '@fastgpt/global/common/string/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -39,8 +42,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
text,
chunkLen: chunkSize,
overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0,
customReg: chunkSplitter ? [chunkSplitter] : [],
countTokens: false
customReg: chunkSplitter ? [chunkSplitter] : []
});
// 2. check dataset limit
@@ -67,7 +69,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// 4. push chunks to training queue
const insertResults = await pushDataToDatasetCollection({
const insertResults = await pushDataToTrainingQueue({
teamId,
tmbId,
collectionId,

17
projects/app/src/pages/api/core/dataset/collection/delete.ts
View File

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('CollectionIdId is required');
}
await authDatasetCollection({
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
@@ -24,13 +24,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// find all delete id
const collections = await findCollectionAndChild(collectionId, '_id fileId');
const delIdList = collections.map((item) => item._id);
const collections = await findCollectionAndChild({
teamId,
datasetId: collection.datasetId._id,
collectionId,
fields: '_id teamId fileId metadata'
});
// delete
await delCollectionRelevantData({
collectionIds: delIdList,
fileIds: collections.map((item) => item?.fileId || '').filter(Boolean)
await delCollectionAndRelatedSources({
collections
});
jsonRes(res);

10
projects/app/src/pages/api/core/dataset/collection/list.ts
View File

@@ -7,7 +7,7 @@ import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.
import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
import { PagingData } from '@/types';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { startQueue } from '@/service/utils/tools';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
@@ -87,7 +87,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{
$match: {
$expr: {
$eq: ['$collectionId', '$$id']
$and: [{ $eq: ['$teamId', match.teamId] }, { $eq: ['$collectionId', '$$id'] }]
}
}
},
@@ -105,7 +105,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{
$match: {
$expr: {
$eq: ['$collectionId', '$$id']
$and: [
{ $eq: ['$teamId', match.teamId] },
{ $eq: ['$datasetId', match.datasetId] },
{ $eq: ['$collectionId', '$$id'] }
]
}
}
},

11
projects/app/src/pages/api/core/dataset/collection/sync/link.ts
View File

@@ -6,11 +6,11 @@ import {
getCollectionAndRawText,
reloadCollectionChunks
} from '@fastgpt/service/core/dataset/collection/utils';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller';
import {
DatasetCollectionSyncResultEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constant';
} from '@fastgpt/global/core/dataset/constants';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
@@ -27,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('CollectionIdId is required');
}
const { collection, tmbId } = await authDatasetCollection({
const { collection, teamId, tmbId } = await authDatasetCollection({
req,
authToken: true,
collectionId,
@@ -87,9 +87,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// delete old collection
await delCollectionRelevantData({
collectionIds: [collection._id],
fileIds: collection.fileId ? [collection.fileId] : []
await delCollectionAndRelatedSources({
collections: [collection]
});
jsonRes(res, {

2
projects/app/src/pages/api/core/dataset/create.ts
View File

@@ -5,7 +5,7 @@ import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import { createDefaultCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {

20
projects/app/src/pages/api/core/dataset/data/delete.ts
View File

@@ -3,7 +3,8 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { delDatasetDataByDataId } from '@fastgpt/service/core/dataset/data/controller';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -17,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
}
// 凭证校验
const { datasetData } = await authDatasetData({
const { teamId, datasetData } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
@@ -25,11 +26,20 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
per: 'w'
});
await delDatasetDataByDataId({
collectionId: datasetData.collectionId,
mongoDataId: dataId
// update mongo data update time
await MongoDatasetData.findByIdAndUpdate(dataId, {
updateTime: new Date()
});
// delete vector data
await deleteDatasetDataVector({
teamId,
idList: datasetData.indexes.map((item) => item.dataId)
});
// delete mongo data
await MongoDatasetData.findByIdAndDelete(dataId);
jsonRes(res, {
data: 'success'
});

5
projects/app/src/pages/api/core/dataset/data/insertData.ts
View File

@@ -71,12 +71,13 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// Duplicate data check
await hasSameValue({
teamId,
collectionId,
q: formatQ,
a: formatA
});
const { insertId, tokens } = await insertData2Dataset({
const { insertId, charsLength } = await insertData2Dataset({
teamId,
tmbId,
datasetId,
@@ -91,7 +92,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model: vectorModelData.model
});

10
projects/app/src/pages/api/core/dataset/data/list.ts
View File

@@ -20,11 +20,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
pageSize = Math.min(pageSize, 30);
// 凭证校验
await authDatasetCollection({ req, authToken: true, authApiKey: true, collectionId, per: 'r' });
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'r'
});
searchText = searchText.replace(/'/g, '');
const match = {
teamId,
datasetId: collection.datasetId._id,
collectionId,
...(searchText
? {

5
projects/app/src/pages/api/core/dataset/data/pushData.ts
View File

@@ -3,13 +3,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
import type { PushDatasetDataProps } from '@/global/core/dataset/api.d';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller';
import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -41,7 +40,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
jsonRes<PushDataResponse>(res, {
data: await pushDataToDatasetCollection({
data: await pushDataToTrainingQueue({
...req.body,
teamId,
tmbId

4
projects/app/src/pages/api/core/dataset/data/update.ts
View File

@@ -31,7 +31,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// auth team balance
await authTeamBalance(teamId);
const { tokens } = await updateData2Dataset({
const { charsLength } = await updateData2Dataset({
dataId: id,
q,
a,
@@ -42,7 +42,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model: vectorModel
});

19
projects/app/src/pages/api/core/dataset/delete.ts
View File

@@ -2,32 +2,35 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller';
import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id } = req.query as {
const { id: datasetId } = req.query as {
id: string;
};
if (!id) {
if (!datasetId) {
throw new Error('缺少参数');
}
// auth owner
await authDataset({ req, authToken: true, datasetId: id, per: 'owner' });
const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'owner' });
const deletedIds = await findDatasetIdTreeByTopDatasetId(id);
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId
});
// delete all dataset.data and pg data
await delDatasetRelevantData({ datasetIds: deletedIds });
await delDatasetRelevantData({ datasets });
// delete dataset data
await MongoDataset.deleteMany({
_id: { $in: deletedIds }
_id: { $in: datasets.map((d) => d._id) }
});
jsonRes(res);

11
projects/app/src/pages/api/core/dataset/exportAll.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { addLog } from '@fastgpt/service/common/system/log';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import {
checkExportDatasetLimit,
@@ -30,7 +30,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
limitMinutes: global.feConfigs?.limit?.exportDatasetLimitMinutes
});
const exportIds = await findDatasetIdTreeByTopDatasetId(datasetId);
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId,
fields: '_id'
});
res.setHeader('Content-Type', 'text/csv; charset=utf-8;');
res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; ');
@@ -42,7 +46,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
a: string;
}>(
{
datasetId: { $in: exportIds }
teamId,
datasetId: { $in: datasets.map((d) => d._id) }
},
'q a'
)

2
projects/app/src/pages/api/core/dataset/list.ts
View File

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';

5
projects/app/src/pages/api/core/dataset/searchTest.ts
View File

@@ -47,7 +47,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// model: global.chatModels[0].model
// });
const { searchRes, tokens, ...result } = await searchDatasetData({
const { searchRes, charsLength, ...result } = await searchDatasetData({
teamId,
rawQuery: text,
queries: [text],
model: dataset.vectorModel,
@@ -62,7 +63,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model: dataset.vectorModel,
source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt
});

2
projects/app/src/pages/api/plugins/customFeedback/index.ts
View File

@@ -12,6 +12,7 @@ type Props = HttpBodyType<{
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const {
appId,
chatId,
responseChatItemId: chatItemId,
data: { defaultFeedback, customFeedback }
@@ -30,6 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// wait the chat finish
setTimeout(() => {
addCustomFeedbacks({
appId,
chatId,
chatItemId,
feedbacks: [feedback]

11
projects/app/src/pages/api/support/user/account/loginByPassword.ts
View File

@@ -10,7 +10,7 @@ import { UserStatusEnum } from '@fastgpt/global/support/user/constant';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { username, password, tmbId = '' } = req.body as PostLoginProps;
const { username, password } = req.body as PostLoginProps;
if (!username || !password) {
throw new Error('缺少参数');
@@ -40,7 +40,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('密码错误');
}
const userDetail = await getUserDetail({ tmbId, userId: user._id });
const userDetail = await getUserDetail({
tmbId: user?.lastLoginTmbId,
userId: user._id
});
MongoUser.findByIdAndUpdate(user._id, {
lastLoginTmbId: userDetail.team.tmbId
});
const token = createJWT(userDetail);
setCookie(res, token);

37
projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts Normal file
View File

@@ -0,0 +1,37 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { size } = req.query as {
size: string;
};
// 凭证校验
const { teamId } = await authCert({ req, authToken: true });
if (!size) {
return jsonRes(res);
}
const numberSize = Number(size);
await checkDatasetLimit({
teamId,
freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize,
insertLen: numberSize
});
jsonRes(res);
} catch (err) {
res.status(500);
jsonRes(res, {
code: 500,
error: err
});
}
}

19
projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts
View File

@@ -1,29 +1,32 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { CreateTrainingBillProps } from '@fastgpt/global/support/wallet/bill/api.d';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { name, vectorModel, agentModel } = req.body as CreateTrainingBillProps;
const { name, datasetId } = req.body as CreateTrainingBillProps;
const { teamId, tmbId } = await authCert({ req, authToken: true, authApiKey: true });
const vectorModelData = getVectorModel(vectorModel);
const agentModelData = getQAModel(agentModel);
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'w'
});
const { billId } = await createTrainingBill({
teamId,
tmbId,
appName: name,
billSource: BillSourceEnum.training,
vectorModel: vectorModelData.name,
agentModel: agentModelData.name
vectorModel: getVectorModel(dataset.vectorModel).name,
agentModel: getQAModel(dataset.agentModel).name
});
jsonRes<string>(res, {

8
projects/app/src/pages/api/v1/audio/transcriptions.ts
View File

@@ -17,11 +17,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
try {
const {
files,
metadata: { duration, shareId }
file,
data: { duration }
} = await upload.doUpload<{ duration: number; shareId?: string }>(req, res);
filePaths = files.map((file) => file.path);
filePaths = [file.path];
const { teamId, tmbId } = await authCert({ req, authToken: true });
@@ -29,8 +29,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
throw new Error('whisper model not found');
}
const file = files[0];
if (!file) {
throw new Error('file not found');
}

7
projects/app/src/pages/api/v1/chat/completions.ts
View File

@@ -195,7 +195,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
// get and concat history
const { history } = await getChatItems({ chatId, limit: 30, field: `dataId obj value` });
const { history } = await getChatItems({
appId: app._id,
chatId,
limit: 30,
field: `dataId obj value`
});
const concatHistories = history.concat(chatMessages);
const responseChatItemId: string | undefined = messages[messages.length - 1].dataId;

8
projects/app/src/pages/api/v1/embeddings.ts
View File

@@ -33,7 +33,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
await authTeamBalance(teamId);
const { tokens, vectors } = await getVectorsByText({ input: query, model });
const { charsLength, vectors } = await getVectorsByText({ input: query, model });
res.json({
object: 'list',
@@ -44,15 +44,15 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
})),
model,
usage: {
prompt_tokens: tokens,
total_tokens: tokens
prompt_tokens: charsLength,
total_tokens: charsLength
}
});
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model,
billId,
source: getBillSourceByAuthType({ authType })