gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

4.6.7-alpha commit (#743)

Browse Source 
Co-authored-by: Archer <545436317@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-01-19 11:17:28 +08:00
committed by GitHub
parent 8ee7407c4c
commit c031e6dcc9
324 changed files with 8509 additions and 4757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
View File

@@ -7,7 +7,7 @@ import { jsonRes } from '@fastgpt/service/common/response';
import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d';
import type { ModuleItemType } from '@fastgpt/global/core/module/type';
import { FormatForm2ModulesProps } from '@fastgpt/global/core/app/api';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import { getExtractModel } from '@/service/core/ai/model';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -37,7 +37,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
return [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -49,7 +49,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -58,7 +58,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -93,7 +93,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -189,7 +189,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -268,7 +268,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
const modules: ModuleItemType[] = [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -280,7 +280,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -289,17 +289,13 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
{
moduleId: 'vuc92c',
key: 'userChatInput'
},
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
@@ -307,7 +303,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
},
{
moduleId: 'datasetSearch',
name: '知识库搜索',
name: 'core.module.template.Dataset search',
avatar: '/imgs/module/db.png',
flowType: 'datasetSearchNode',
showStatus: true,
@@ -447,6 +443,18 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
valueType: 'boolean',
type: 'source',
targets: []
},
{
key: 'userChatInput',
label: 'core.module.input.label.user question',
type: 'hidden',
valueType: 'string',
targets: [
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
]
},
@@ -473,7 +481,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -569,7 +577,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:

38
projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts
View File

@@ -33,7 +33,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
return [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -45,7 +45,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -54,7 +54,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -89,7 +89,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -185,7 +185,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -264,7 +264,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
const modules: ModuleItemType[] = [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -276,7 +276,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -285,17 +285,13 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
{
moduleId: 'vuc92c',
key: 'userChatInput'
},
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
@@ -303,7 +299,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
},
{
moduleId: 'datasetSearch',
name: '知识库搜索',
name: 'core.module.template.Dataset search',
avatar: '/imgs/module/db.png',
flowType: 'datasetSearchNode',
showStatus: true,
@@ -457,6 +453,18 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
valueType: 'boolean',
type: 'source',
targets: []
},
{
key: 'userChatInput',
label: 'core.module.input.label.user question',
type: 'hidden',
valueType: 'string',
targets: [
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
]
},
@@ -483,7 +491,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -579,7 +587,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:

19
projects/app/src/pages/api/core/app/getChatLogs.ts
View File

@@ -8,6 +8,7 @@ import { Types } from '@fastgpt/service/common/mongo';
import { addDays } from 'date-fns';
import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -28,8 +29,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { teamId } = await authApp({ req, authToken: true, appId, per: 'w' });
const where = {
appId: new Types.ObjectId(appId),
teamId: new Types.ObjectId(teamId),
appId: new Types.ObjectId(appId),
updateTime: {
$gte: new Date(dateStart),
$lte: new Date(dateEnd)
@@ -41,18 +42,26 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
{ $match: where },
{
$lookup: {
from: 'chatitems',
let: { chat_id: '$chatId' },
from: ChatItemCollectionName,
let: { chatId: '$chatId' },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$chatId', '$$chat_id'] },
{ $eq: ['$appId', new Types.ObjectId(appId)] }
{ $eq: ['$appId', new Types.ObjectId(appId)] },
{ $eq: ['$chatId', '$$chatId'] }
]
}
}
},
{
$project: {
userGoodFeedback: 1,
userBadFeedback: 1,
customFeedbacks: 1,
adminFeedback: 1
}
}
],
as: 'chatitems'

3
projects/app/src/pages/api/core/chat/clearHistories.ts
View File

@@ -35,16 +35,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
return Promise.reject('Param are error');
})();
console.log(match);
// find chatIds
const list = await MongoChat.find(match, 'chatId').lean();
const idList = list.map((item) => item.chatId);
await MongoChatItem.deleteMany({
appId,
chatId: { $in: idList }
});
await MongoChat.deleteMany({
appId,
chatId: { $in: idList }
});

2
projects/app/src/pages/api/core/chat/delHistory.ts
View File

@@ -23,9 +23,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
await MongoChatItem.deleteMany({
appId,
chatId
});
await MongoChat.findOneAndRemove({
appId,
chatId
});

2
projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts
View File

@@ -27,6 +27,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await MongoChatItem.findOneAndUpdate(
{
appId,
chatId,
dataId: chatItemId
},
{

13
projects/app/src/pages/api/core/chat/feedback/closeCustom.ts
View File

@@ -2,14 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import type {
AdminUpdateFeedbackParams,
CloseCustomFeedbackParams
} from '@/global/core/chat/api.d';
import type { CloseCustomFeedbackParams } from '@/global/core/chat/api.d';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { autChatCrud } from '@/service/support/permission/auth/chat';
/* 初始化我的聊天框，需要身份验证 */
/* remove custom feedback */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
@@ -29,13 +26,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await authCert({ req, authToken: true });
await MongoChatItem.findOneAndUpdate(
{ dataId: chatItemId },
{ appId, chatId, dataId: chatItemId },
{ $unset: { [`customFeedbacks.${index}`]: 1 } }
);
await MongoChatItem.findOneAndUpdate(
{ dataId: chatItemId },
{ $pull: { customFeedbacks: null } }
);
jsonRes(res);
} catch (err) {

1
projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts
View File

@@ -29,6 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await MongoChatItem.findOneAndUpdate(
{
appId,
chatId,
dataId: chatItemId
},

2
projects/app/src/pages/api/core/chat/getHistories.ts
View File

@@ -31,8 +31,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
if (appId) {
const { tmbId } = await authCert({ req, authToken: true });
return {
appId,
tmbId,
appId,
source: ChatSourceEnum.online
};
}

3
projects/app/src/pages/api/core/chat/init.ts
View File

@@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
appId,
per: 'r'
}),
chatId ? MongoChat.findOne({ chatId }) : undefined
chatId ? MongoChat.findOne({ appId, chatId }) : undefined
]);
// auth chat permission
@@ -41,6 +41,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// get app and history
const { history } = await getChatItems({
appId,
chatId,
limit: 30,
field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${

5
projects/app/src/pages/api/core/chat/item/delete.ts
View File

@@ -25,8 +25,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
await MongoChatItem.deleteOne({
dataId: contentId,
chatId
appId,
chatId,
dataId: contentId
});
jsonRes(res);

2
projects/app/src/pages/api/core/chat/item/getSpeech.ts
View File

@@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
try {
pushAudioSpeechBill({
model: model,
textLen: input.length,
charsLength: input.length,
tmbId,
teamId,
source: authType2BillSource({ authType })

3
projects/app/src/pages/api/core/chat/outLink/init.ts
View File

@@ -26,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// auth app permission
const [tmb, chat, app] = await Promise.all([
MongoTeamMember.findById(shareChat.tmbId, '_id userId').populate('userId', 'avatar').lean(),
MongoChat.findOne({ chatId, shareId }).lean(),
MongoChat.findOne({ appId, chatId, shareId }).lean(),
MongoApp.findById(appId).lean()
]);
@@ -40,6 +40,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
const { history } = await getChatItems({
appId: app._id,
chatId,
limit: 30,
field: `dataId obj value userGoodFeedback userBadFeedback ${

2
projects/app/src/pages/api/core/chat/updateHistory.ts
View File

@@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
await MongoChat.findOneAndUpdate(
{ chatId },
{ appId, chatId },
{
...(customTitle !== undefined && { customTitle }),
...(top !== undefined && { top })

2
projects/app/src/pages/api/core/dataset/allDataset.ts
View File

@@ -6,7 +6,7 @@ import { getVectorModel } from '@/service/core/ai/model';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
/* get all dataset by teamId or tmbId */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

85
projects/app/src/pages/api/core/dataset/collection/create/file.ts Normal file
View File

@@ -0,0 +1,85 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller';
import { getUploadModel } from '@fastgpt/service/common/file/multer';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { FileCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
/**
* Creates the multer uploader
*/
const upload = getUploadModel({
maxSize: 500 * 1024 * 1024
});
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let filePaths: string[] = [];
const { datasetId } = req.query as { datasetId: string };
try {
await connectToDatabase();
const { teamId, tmbId } = await authDataset({
req,
authToken: true,
authApiKey: true,
per: 'w',
datasetId
});
const { file, bucketName, data } = await upload.doUpload<FileCreateDatasetCollectionParams>(
req,
res
);
filePaths = [file.path];
if (!file || !bucketName) {
throw new Error('file is empty');
}
const { fileMetadata, collectionMetadata, ...collectionData } = data;
// upload file and create collection
const fileId = await uploadFile({
teamId,
tmbId,
bucketName,
path: file.path,
filename: file.originalname,
contentType: file.mimetype,
metadata: fileMetadata
});
// create collection
const collectionId = await createOneCollection({
...collectionData,
metadata: collectionMetadata,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.file,
fileId
});
jsonRes(res, {
data: collectionId
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
removeFilesByPaths(filePaths);
}
export const config = {
api: {
bodyParser: false
}
};

5
projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts → projects/app/src/pages/api/core/dataset/collection/create/link.ts
View File

@@ -7,7 +7,10 @@ import { connectToDatabase } from '@/service/mongo';
import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';

12
projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts → projects/app/src/pages/api/core/dataset/collection/create/text.ts
View File

@@ -7,11 +7,14 @@ import { connectToDatabase } from '@/service/mongo';
import type { TextCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller';
import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller';
import { hashStr } from '@fastgpt/global/common/string/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -39,8 +42,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
text,
chunkLen: chunkSize,
overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0,
customReg: chunkSplitter ? [chunkSplitter] : [],
countTokens: false
customReg: chunkSplitter ? [chunkSplitter] : []
});
// 2. check dataset limit
@@ -67,7 +69,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// 4. push chunks to training queue
const insertResults = await pushDataToDatasetCollection({
const insertResults = await pushDataToTrainingQueue({
teamId,
tmbId,
collectionId,

17
projects/app/src/pages/api/core/dataset/collection/delete.ts
View File

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('CollectionIdId is required');
}
await authDatasetCollection({
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
@@ -24,13 +24,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// find all delete id
const collections = await findCollectionAndChild(collectionId, '_id fileId');
const delIdList = collections.map((item) => item._id);
const collections = await findCollectionAndChild({
teamId,
datasetId: collection.datasetId._id,
collectionId,
fields: '_id teamId fileId metadata'
});
// delete
await delCollectionRelevantData({
collectionIds: delIdList,
fileIds: collections.map((item) => item?.fileId || '').filter(Boolean)
await delCollectionAndRelatedSources({
collections
});
jsonRes(res);

10
projects/app/src/pages/api/core/dataset/collection/list.ts
View File

@@ -7,7 +7,7 @@ import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.
import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
import { PagingData } from '@/types';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { startQueue } from '@/service/utils/tools';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
@@ -87,7 +87,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{
$match: {
$expr: {
$eq: ['$collectionId', '$$id']
$and: [{ $eq: ['$teamId', match.teamId] }, { $eq: ['$collectionId', '$$id'] }]
}
}
},
@@ -105,7 +105,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{
$match: {
$expr: {
$eq: ['$collectionId', '$$id']
$and: [
{ $eq: ['$teamId', match.teamId] },
{ $eq: ['$datasetId', match.datasetId] },
{ $eq: ['$collectionId', '$$id'] }
]
}
}
},

11
projects/app/src/pages/api/core/dataset/collection/sync/link.ts
View File

@@ -6,11 +6,11 @@ import {
getCollectionAndRawText,
reloadCollectionChunks
} from '@fastgpt/service/core/dataset/collection/utils';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller';
import {
DatasetCollectionSyncResultEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constant';
} from '@fastgpt/global/core/dataset/constants';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
@@ -27,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('CollectionIdId is required');
}
const { collection, tmbId } = await authDatasetCollection({
const { collection, teamId, tmbId } = await authDatasetCollection({
req,
authToken: true,
collectionId,
@@ -87,9 +87,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// delete old collection
await delCollectionRelevantData({
collectionIds: [collection._id],
fileIds: collection.fileId ? [collection.fileId] : []
await delCollectionAndRelatedSources({
collections: [collection]
});
jsonRes(res, {

2
projects/app/src/pages/api/core/dataset/create.ts
View File

@@ -5,7 +5,7 @@ import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import { createDefaultCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {

20
projects/app/src/pages/api/core/dataset/data/delete.ts
View File

@@ -3,7 +3,8 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { delDatasetDataByDataId } from '@fastgpt/service/core/dataset/data/controller';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -17,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
}
// 凭证校验
const { datasetData } = await authDatasetData({
const { teamId, datasetData } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
@@ -25,11 +26,20 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
per: 'w'
});
await delDatasetDataByDataId({
collectionId: datasetData.collectionId,
mongoDataId: dataId
// update mongo data update time
await MongoDatasetData.findByIdAndUpdate(dataId, {
updateTime: new Date()
});
// delete vector data
await deleteDatasetDataVector({
teamId,
idList: datasetData.indexes.map((item) => item.dataId)
});
// delete mongo data
await MongoDatasetData.findByIdAndDelete(dataId);
jsonRes(res, {
data: 'success'
});

5
projects/app/src/pages/api/core/dataset/data/insertData.ts
View File

@@ -71,12 +71,13 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// Duplicate data check
await hasSameValue({
teamId,
collectionId,
q: formatQ,
a: formatA
});
const { insertId, tokens } = await insertData2Dataset({
const { insertId, charsLength } = await insertData2Dataset({
teamId,
tmbId,
datasetId,
@@ -91,7 +92,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model: vectorModelData.model
});

10
projects/app/src/pages/api/core/dataset/data/list.ts
View File

@@ -20,11 +20,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
pageSize = Math.min(pageSize, 30);
// 凭证校验
await authDatasetCollection({ req, authToken: true, authApiKey: true, collectionId, per: 'r' });
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'r'
});
searchText = searchText.replace(/'/g, '');
const match = {
teamId,
datasetId: collection.datasetId._id,
collectionId,
...(searchText
? {

5
projects/app/src/pages/api/core/dataset/data/pushData.ts
View File

@@ -3,13 +3,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
import type { PushDatasetDataProps } from '@/global/core/dataset/api.d';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller';
import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -41,7 +40,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
jsonRes<PushDataResponse>(res, {
data: await pushDataToDatasetCollection({
data: await pushDataToTrainingQueue({
...req.body,
teamId,
tmbId

4
projects/app/src/pages/api/core/dataset/data/update.ts
View File

@@ -31,7 +31,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// auth team balance
await authTeamBalance(teamId);
const { tokens } = await updateData2Dataset({
const { charsLength } = await updateData2Dataset({
dataId: id,
q,
a,
@@ -42,7 +42,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model: vectorModel
});

19
projects/app/src/pages/api/core/dataset/delete.ts
View File

@@ -2,32 +2,35 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller';
import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id } = req.query as {
const { id: datasetId } = req.query as {
id: string;
};
if (!id) {
if (!datasetId) {
throw new Error('缺少参数');
}
// auth owner
await authDataset({ req, authToken: true, datasetId: id, per: 'owner' });
const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'owner' });
const deletedIds = await findDatasetIdTreeByTopDatasetId(id);
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId
});
// delete all dataset.data and pg data
await delDatasetRelevantData({ datasetIds: deletedIds });
await delDatasetRelevantData({ datasets });
// delete dataset data
await MongoDataset.deleteMany({
_id: { $in: deletedIds }
_id: { $in: datasets.map((d) => d._id) }
});
jsonRes(res);

11
projects/app/src/pages/api/core/dataset/exportAll.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { addLog } from '@fastgpt/service/common/system/log';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import {
checkExportDatasetLimit,
@@ -30,7 +30,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
limitMinutes: global.feConfigs?.limit?.exportDatasetLimitMinutes
});
const exportIds = await findDatasetIdTreeByTopDatasetId(datasetId);
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId,
fields: '_id'
});
res.setHeader('Content-Type', 'text/csv; charset=utf-8;');
res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; ');
@@ -42,7 +46,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
a: string;
}>(
{
datasetId: { $in: exportIds }
teamId,
datasetId: { $in: datasets.map((d) => d._id) }
},
'q a'
)

2
projects/app/src/pages/api/core/dataset/list.ts
View File

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';

5
projects/app/src/pages/api/core/dataset/searchTest.ts
View File

@@ -47,7 +47,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// model: global.chatModels[0].model
// });
const { searchRes, tokens, ...result } = await searchDatasetData({
const { searchRes, charsLength, ...result } = await searchDatasetData({
teamId,
rawQuery: text,
queries: [text],
model: dataset.vectorModel,
@@ -62,7 +63,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model: dataset.vectorModel,
source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt
});