gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V4.6.7-production (#759)

Browse Source
This commit is contained in:
Archer
2024-01-22 13:48:55 +08:00
committed by GitHub
parent 91b7d81c1a
commit aab6ee51eb
41 changed files with 1777 additions and 2113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
projects/app/src/pages/api/core/dataset/collection/create/file.ts
View File

@@ -1,85 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller';
import { getUploadModel } from '@fastgpt/service/common/file/multer';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { FileCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
/**
* Creates the multer uploader
*/
const upload = getUploadModel({
maxSize: 500 * 1024 * 1024
});
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let filePaths: string[] = [];
const { datasetId } = req.query as { datasetId: string };
try {
await connectToDatabase();
const { teamId, tmbId } = await authDataset({
req,
authToken: true,
authApiKey: true,
per: 'w',
datasetId
});
const { file, bucketName, data } = await upload.doUpload<FileCreateDatasetCollectionParams>(
req,
res
);
filePaths = [file.path];
if (!file || !bucketName) {
throw new Error('file is empty');
}
const { fileMetadata, collectionMetadata, ...collectionData } = data;
// upload file and create collection
const fileId = await uploadFile({
teamId,
tmbId,
bucketName,
path: file.path,
filename: file.originalname,
contentType: file.mimetype,
metadata: fileMetadata
});
// create collection
const collectionId = await createOneCollection({
...collectionData,
metadata: collectionMetadata,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.file,
fileId
});
jsonRes(res, {
data: collectionId
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
removeFilesByPaths(filePaths);
}
export const config = {
api: {
bodyParser: false
}
};

91
projects/app/src/pages/api/core/dataset/collection/create/link.ts
View File

@@ -1,91 +0,0 @@
/*
Create one dataset collection
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
import { reloadCollectionChunks } from '@fastgpt/service/core/dataset/collection/utils';
import { startQueue } from '@/service/utils/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
link,
trainingType = TrainingModeEnum.chunk,
chunkSize = 512,
chunkSplitter,
qaPrompt,
...body
} = req.body as LinkCreateDatasetCollectionParams;
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId,
per: 'w'
});
// 1. check dataset limit
await checkDatasetLimit({
teamId,
freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize,
insertLen: predictDataLimitLength(trainingType, new Array(10))
});
// 2. create collection
const collectionId = await createOneCollection({
...body,
name: link,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.link,
trainingType,
chunkSize,
chunkSplitter,
qaPrompt,
rawLink: link
});
// 3. create bill and start sync
const { billId } = await createTrainingBill({
teamId,
tmbId,
appName: 'core.dataset.collection.Sync Collection',
billSource: BillSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel).name,
agentModel: getQAModel(dataset.agentModel).name
});
await reloadCollectionChunks({
collectionId,
tmbId,
billId
});
startQueue();
jsonRes(res, {
data: { collectionId }
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}

117
projects/app/src/pages/api/core/dataset/collection/create/text.ts
View File

@@ -1,117 +0,0 @@
/*
Create one dataset collection
*/
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { TextCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller';
import { hashStr } from '@fastgpt/global/common/string/tools';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const {
name,
text,
trainingType = TrainingModeEnum.chunk,
chunkSize = 512,
chunkSplitter,
qaPrompt,
...body
} = req.body as TextCreateDatasetCollectionParams;
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId: body.datasetId,
per: 'w'
});
// 1. split text to chunks
const { chunks } = splitText2Chunks({
text,
chunkLen: chunkSize,
overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0,
customReg: chunkSplitter ? [chunkSplitter] : []
});
// 2. check dataset limit
await checkDatasetLimit({
teamId,
freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize,
insertLen: predictDataLimitLength(trainingType, chunks)
});
// 3. create collection and training bill
const [collectionId, { billId }] = await Promise.all([
createOneCollection({
...body,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.virtual,
name,
trainingType,
chunkSize,
chunkSplitter,
qaPrompt,
hashRawText: hashStr(text),
rawTextLength: text.length
}),
createTrainingBill({
teamId,
tmbId,
appName: name,
billSource: BillSourceEnum.training,
vectorModel: getVectorModel(dataset.vectorModel)?.name,
agentModel: getQAModel(dataset.agentModel)?.name
})
]);
// 4. push chunks to training queue
const insertResults = await pushDataToTrainingQueue({
teamId,
tmbId,
collectionId,
trainingMode: trainingType,
prompt: qaPrompt,
billId,
data: chunks.map((text, index) => ({
q: text,
chunkIndex: index
}))
});
jsonRes(res, {
data: { collectionId, results: insertResults }
});
} catch (err) {
jsonRes(res, {
code: 500,
error: err
});
}
}
export const config = {
api: {
bodyParser: {
sizeLimit: '10mb'
}
}
};

9
projects/app/src/pages/api/core/dataset/create.ts
View File

@@ -6,6 +6,7 @@ import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import { createDefaultCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -13,18 +14,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const {
parentId,
name,
type,
type = DatasetTypeEnum.dataset,
avatar,
vectorModel = global.vectorModels[0].model,
agentModel = global.qaModels[0].model
} = req.body as CreateDatasetParams;
// auth
const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true });
const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true, authApiKey: true });
// check model valid
const vectorModelStore = global.vectorModels.find((item) => item.model === vectorModel);
const agentModelStore = global.qaModels.find((item) => item.model === agentModel);
const vectorModelStore = getVectorModel(vectorModel);
const agentModelStore = getQAModel(agentModel);
if (!vectorModelStore || !agentModelStore) {
throw new Error('vectorModel or qaModel is invalid');
}

8
projects/app/src/pages/api/core/dataset/delete.ts
View File

@@ -18,7 +18,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
}
// auth owner
const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'owner' });
const { teamId } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'owner'
});
const datasets = await findDatasetAndAllChildren({
teamId,

64
projects/app/src/pages/api/plusApi/[...path].ts
View File

@@ -1,64 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { request } from '@fastgpt/service/common/api/plusRequest';
import type { Method } from 'axios';
import { setCookie } from '@fastgpt/service/support/permission/controller';
import { connectToDatabase } from '@/service/mongo';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const method = (req.method || 'POST') as Method;
const { path = [], ...query } = req.query as any;
const url = `/${path?.join('/')}?${new URLSearchParams(query).toString()}`;
if (!url) {
throw new Error('url is empty');
}
const data = req.body || query;
const repose = await request(
url,
data,
{
headers: {
...req.headers,
// @ts-ignore
rootkey: undefined
}
},
method
);
/* special response */
// response cookie
if (repose?.cookie) {
setCookie(res, repose.cookie);
return jsonRes(res, {
data: repose?.cookie
});
}
jsonRes(res, {
data: repose
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
export const config = {
api: {
bodyParser: {
sizeLimit: '10mb'
},
responseLimit: '10mb'
}
};

56
projects/app/src/pages/api/proApi/[...path].ts Normal file
View File

@@ -0,0 +1,56 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { request } from 'http';
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
import url from 'url';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { path = [], ...query } = req.query as any;
const requestPath = `/api/${path?.join('/')}?${new URLSearchParams(query).toString()}`;
if (!requestPath) {
throw new Error('url is empty');
}
const parsedUrl = url.parse(FastGPTProUrl);
delete req.headers?.rootkey;
const requestResult = request({
protocol: parsedUrl.protocol,
hostname: parsedUrl.hostname,
port: parsedUrl.port,
path: requestPath,
method: req.method,
headers: req.headers
});
req.pipe(requestResult);
requestResult.on('response', (response) => {
Object.keys(response.headers).forEach((key) => {
// @ts-ignore
res.setHeader(key, response.headers[key]);
});
response.statusCode && res.writeHead(response.statusCode);
response.pipe(res);
});
requestResult.on('error', (e) => {
res.send(e);
res.end();
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
}
export const config = {
api: {
bodyParser: false
}
};