V4.9.4 feature (#4470)
* Training status (#4424) * dataset data training state (#4311) * dataset data training state * fix * fix ts * fix * fix api format * fix * fix * perf: count training * format * fix: dataset training state (#4417) * fix * add test * fix * fix * fix test * fix test * perf: training count * count * loading status --------- Co-authored-by: heheer <heheer@sealos.io> * doc * website sync feature (#4429) * perf: introduce BullMQ for website sync (#4403) * perf: introduce BullMQ for website sync * feat: new redis module * fix: remove graceful shutdown * perf: improve UI in dataset detail - Updated the "change" icon SVG file. - Modified i18n strings. - Added new i18n string "immediate_sync". - Improved UI in dataset detail page, including button icons and background colors. * refactor: Add chunkSettings to DatasetSchema * perf: website sync ux * env template * fix: clean up website dataset when updating chunk settings (#4420) * perf: check setting updated * perf: worker currency * feat: init script for website sync refactor (#4425) * website feature doc --------- Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com> * pro migration (#4388) (#4433) * pro migration * reuse customPdfParseType Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com> * perf: remove loading ui * feat: config chat file expired time * Redis cache (#4436) * perf: add Redis cache for vector counting (#4432) * feat: cache * perf: get cache key --------- Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com> * perf: mobile voice input (#4437) * update:Mobile voice interaction (#4362) * Add files via upload * Add files via upload * Update ollama.md * Update ollama.md * Add files via upload * Update useSpeech.ts * Update ChatInput.tsx * Update useSpeech.ts * Update ChatInput.tsx * Update useSpeech.ts * Update constants.ts * Add files via upload * Update ChatInput.tsx * Update useSpeech.ts * Update useSpeech.ts * Update useSpeech.ts * Update ChatInput.tsx * Add files via upload * Update common.json * Update VoiceInput.tsx * Update ChatInput.tsx * Update VoiceInput.tsx * Update useSpeech.ts * Update useSpeech.ts * Update common.json * Update common.json * Update common.json * Update VoiceInput.tsx * Update VoiceInput.tsx * Update ChatInput.tsx * Update VoiceInput.tsx * Update ChatInput.tsx * Update VoiceInput.tsx * Update ChatInput.tsx * Update useSpeech.ts * Update common.json * Update chat.json * Update common.json * Update chat.json * Update common.json * Update chat.json * Update VoiceInput.tsx * Update ChatInput.tsx * Update useSpeech.ts * Update VoiceInput.tsx * speech ui * 优化语音输入组件,调整输入框显示逻辑,修复语音输入遮罩层样式,更新画布背景透明度,增强用户交互体验。 (#4435) * perf: mobil voice input --------- Co-authored-by: dreamer6680 <1468683855@qq.com> * Test completion v2 (#4438) * add v2 completions (#4364) * add v2 completions * completion config * config version * fix * frontend * doc * fix * fix: completions v2 api --------- Co-authored-by: heheer <heheer@sealos.io> * package * Test mongo log (#4443) * feat: mongodb-log (#4426) * perf: mongo log * feat: completions stop reasoner * mongo db log --------- Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com> * update doc * Update doc * fix external var ui (#4444) * action * fix: ts (#4458) * preview doc action add docs preview permission update preview action udpate action * update doc (#4460) * update preview action * update doc * remove * update * schema * update mq export;perf: redis cache (#4465) * perf: redis cache * update mq export * perf: website sync error tip * add error worker * website sync ui (#4466) * Updated the dynamic display of the voice input pop-up (#4469) * Update VoiceInput.tsx * Update VoiceInput.tsx * Update VoiceInput.tsx * fix: voice input --------- Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com> Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com> Co-authored-by: dreamer6680 <1468683855@qq.com> Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
This commit is contained in:
@@ -1,6 +1,5 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { addHours } from 'date-fns';
|
||||
import { MongoImage } from '@fastgpt/service/common/file/image/schema';
|
||||
@@ -56,7 +55,6 @@ async function checkInvalidImg(start: Date, end: Date, limit = 50) {
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
await authCert({ req, authRoot: true });
|
||||
const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number };
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
|
||||
import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
|
||||
@@ -8,7 +7,6 @@ import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
await authCert({ req, authRoot: true });
|
||||
|
||||
await MongoPlugin.updateMany(
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
|
||||
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
await authCert({ req, authRoot: true });
|
||||
|
||||
// 删除索引
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
|
||||
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
|
||||
@@ -9,7 +8,6 @@ import { POST } from '@fastgpt/service/common/api/plusRequest';
|
||||
/* 初始化发布的版本 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
await authCert({ req, authRoot: true });
|
||||
|
||||
await MongoAppVersion.updateMany(
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { addHours } from 'date-fns';
|
||||
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
|
||||
@@ -174,7 +173,6 @@ const checkInvalidDataText = async () => {
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
await authCert({ req, authRoot: true });
|
||||
const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number };
|
||||
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
|
||||
import { MongoApp } from '@fastgpt/service/core/app/schema';
|
||||
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
await authCert({ req, authRoot: true });
|
||||
|
||||
await MongoApp.updateMany(
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
|
||||
import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant';
|
||||
@@ -8,7 +7,6 @@ import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/
|
||||
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
await authCert({ req, authRoot: true });
|
||||
|
||||
await MongoDataset.updateMany(
|
||||
|
||||
53
projects/app/src/pages/api/admin/initv494.ts
Normal file
53
projects/app/src/pages/api/admin/initv494.ts
Normal file
@@ -0,0 +1,53 @@
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { retryFn } from '@fastgpt/global/common/system/utils';
|
||||
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
|
||||
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
|
||||
import { upsertWebsiteSyncJobScheduler } from '@fastgpt/service/core/dataset/websiteSync';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { addHours } from 'date-fns';
|
||||
import { NextApiRequest, NextApiResponse } from 'next';
|
||||
|
||||
const initWebsiteSyncData = async () => {
|
||||
// find out all website dataset
|
||||
const datasets = await MongoDataset.find({ type: DatasetTypeEnum.websiteDataset }).lean();
|
||||
|
||||
console.log('更新站点同步的定时器');
|
||||
// Add scheduler for all website dataset
|
||||
await Promise.all(
|
||||
datasets.map((dataset) => {
|
||||
if (dataset.autoSync) {
|
||||
// 随机生成一个往后 1~24 小时的时间
|
||||
const time = addHours(new Date(), Math.floor(Math.random() * 23) + 1);
|
||||
return retryFn(() =>
|
||||
upsertWebsiteSyncJobScheduler({ datasetId: String(dataset._id) }, time.getTime())
|
||||
);
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
console.log('移除站点同步集合的定时器');
|
||||
// Remove all nextSyncTime
|
||||
await retryFn(() =>
|
||||
MongoDatasetCollection.updateMany(
|
||||
{
|
||||
teamId: datasets.map((dataset) => dataset.teamId),
|
||||
datasetId: datasets.map((dataset) => dataset._id)
|
||||
},
|
||||
{
|
||||
$unset: {
|
||||
nextSyncTime: 1
|
||||
}
|
||||
}
|
||||
)
|
||||
);
|
||||
};
|
||||
async function handler(req: NextApiRequest, _res: NextApiResponse) {
|
||||
await authCert({ req, authRoot: true });
|
||||
|
||||
await initWebsiteSyncData();
|
||||
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
export default NextAPI(handler);
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authFileToken } from '@fastgpt/service/support/permission/controller';
|
||||
import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
|
||||
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
|
||||
@@ -23,8 +22,6 @@ const previewableExtensions = [
|
||||
// Abandoned, use: file/read/[filename].ts
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
|
||||
const { token } = req.query as { token: string };
|
||||
|
||||
const { fileId, bucketName } = await authFileToken(token);
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { authFileToken } from '@fastgpt/service/support/permission/controller';
|
||||
import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
|
||||
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
|
||||
@@ -21,8 +20,6 @@ const previewableExtensions = [
|
||||
];
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
|
||||
const { token, filename } = req.query as { token: string; filename: string };
|
||||
|
||||
const { fileId, bucketName } = await authFileToken(token);
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller';
|
||||
import { UploadImgProps } from '@fastgpt/global/common/file/api';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
@@ -9,7 +8,6 @@ import { NextAPI } from '@/service/middleware/entry';
|
||||
Upload avatar image
|
||||
*/
|
||||
async function handler(req: NextApiRequest, res: NextApiResponse): Promise<string> {
|
||||
await connectToDatabase();
|
||||
const body = req.body as UploadImgProps;
|
||||
|
||||
const { teamId } = await authCert({ req, authToken: true });
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { startTrainingQueue } from '@/service/core/dataset/training/utils';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
await authCert({ req, authToken: true });
|
||||
startTrainingQueue();
|
||||
} catch (error) {}
|
||||
|
||||
@@ -2,13 +2,12 @@
|
||||
import { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
|
||||
import { urlsFetch } from '@fastgpt/service/common/string/cheerio';
|
||||
|
||||
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
let { urlList = [], selector } = req.body as UrlFetchParams;
|
||||
|
||||
if (!urlList || urlList.length === 0) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { pushQuestionGuideUsage } from '@/service/support/wallet/usage/push';
|
||||
import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
@@ -27,7 +27,6 @@ async function handler(
|
||||
res: NextApiResponse<any>
|
||||
) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
const { messages } = req.body;
|
||||
|
||||
const { tmbId, teamId } = await authChatCert({
|
||||
|
||||
@@ -182,7 +182,8 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
histories: newHistories,
|
||||
stream: true,
|
||||
maxRunTimes: WORKFLOW_MAX_RUN_TIMES,
|
||||
workflowStreamResponse: workflowResponseWrite
|
||||
workflowStreamResponse: workflowResponseWrite,
|
||||
version: 'v2'
|
||||
});
|
||||
|
||||
workflowResponseWrite({
|
||||
@@ -197,11 +198,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
event: SseResponseEventEnum.answer,
|
||||
data: '[DONE]'
|
||||
});
|
||||
responseWrite({
|
||||
res,
|
||||
event: SseResponseEventEnum.flowResponses,
|
||||
data: JSON.stringify(flowResponses)
|
||||
});
|
||||
|
||||
// save chat
|
||||
const isInteractiveRequest = !!getLastInteractiveValue(histories);
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import type { AdminUpdateFeedbackParams } from '@/global/core/chat/api.d';
|
||||
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
|
||||
import { authChatCrud } from '@/service/support/permission/auth/chat';
|
||||
@@ -8,7 +8,6 @@ import { authChatCrud } from '@/service/support/permission/auth/chat';
|
||||
/* 初始化我的聊天框,需要身份验证 */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
const { appId, chatId, dataId, datasetId, feedbackDataId, q, a } =
|
||||
req.body as AdminUpdateFeedbackParams;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import type { CloseCustomFeedbackParams } from '@/global/core/chat/api.d';
|
||||
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
|
||||
@@ -10,7 +10,6 @@ import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
|
||||
/* remove custom feedback */
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
const { appId, chatId, dataId, index } = req.body as CloseCustomFeedbackParams;
|
||||
|
||||
if (!dataId || !appId || !chatId) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { GetChatSpeechProps } from '@/global/core/chat/api.d';
|
||||
import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
|
||||
import { pushAudioSpeechUsage } from '@/service/support/wallet/usage/push';
|
||||
@@ -17,7 +17,6 @@ import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
*/
|
||||
async function handler(req: ApiRequestProps<GetChatSpeechProps>, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
const { ttsConfig, input } = req.body;
|
||||
|
||||
if (!ttsConfig.model || !ttsConfig.voice) {
|
||||
|
||||
@@ -12,6 +12,9 @@ import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
|
||||
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
|
||||
import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils';
|
||||
import { getVectorCountByCollectionId } from '@fastgpt/service/common/vectorStore/controller';
|
||||
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
|
||||
import { Types } from 'mongoose';
|
||||
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
|
||||
|
||||
async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType> {
|
||||
const { id } = req.query as { id: string };
|
||||
@@ -30,11 +33,21 @@ async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType>
|
||||
});
|
||||
|
||||
// get file
|
||||
const [file, indexAmount] = await Promise.all([
|
||||
const [file, indexAmount, errorCount] = await Promise.all([
|
||||
collection?.fileId
|
||||
? await getFileById({ bucketName: BucketNameEnum.dataset, fileId: collection.fileId })
|
||||
: undefined,
|
||||
getVectorCountByCollectionId(collection.teamId, collection.datasetId, collection._id)
|
||||
getVectorCountByCollectionId(collection.teamId, collection.datasetId, collection._id),
|
||||
MongoDatasetTraining.countDocuments(
|
||||
{
|
||||
teamId: collection.teamId,
|
||||
datasetId: collection.datasetId,
|
||||
collectionId: id,
|
||||
errorMsg: { $exists: true },
|
||||
retryCount: { $lte: 0 }
|
||||
},
|
||||
readFromSecondary
|
||||
)
|
||||
]);
|
||||
|
||||
return {
|
||||
@@ -46,7 +59,8 @@ async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType>
|
||||
tags: collection.tags
|
||||
}),
|
||||
permission,
|
||||
file
|
||||
file,
|
||||
errorCount
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -93,6 +93,7 @@ async function handler(
|
||||
dataAmount: 0,
|
||||
indexAmount: 0,
|
||||
trainingAmount: 0,
|
||||
hasError: false,
|
||||
permission
|
||||
}))
|
||||
),
|
||||
@@ -113,7 +114,7 @@ async function handler(
|
||||
|
||||
// Compute data amount
|
||||
const [trainingAmount, dataAmount]: [
|
||||
{ _id: string; count: number }[],
|
||||
{ _id: string; count: number; hasError: boolean }[],
|
||||
{ _id: string; count: number }[]
|
||||
] = await Promise.all([
|
||||
MongoDatasetTraining.aggregate(
|
||||
@@ -128,7 +129,8 @@ async function handler(
|
||||
{
|
||||
$group: {
|
||||
_id: '$collectionId',
|
||||
count: { $sum: 1 }
|
||||
count: { $sum: 1 },
|
||||
hasError: { $max: { $cond: [{ $ifNull: ['$errorMsg', false] }, true, false] } }
|
||||
}
|
||||
}
|
||||
],
|
||||
@@ -168,6 +170,7 @@ async function handler(
|
||||
trainingAmount:
|
||||
trainingAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
|
||||
dataAmount: dataAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
|
||||
hasError: trainingAmount.find((amount) => String(amount._id) === String(item._id))?.hasError,
|
||||
permission
|
||||
}))
|
||||
);
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
|
||||
import {
|
||||
DatasetCollectionDataProcessModeEnum,
|
||||
TrainingModeEnum
|
||||
} from '@fastgpt/global/core/dataset/constants';
|
||||
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
|
||||
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
|
||||
type getTrainingDetailParams = {
|
||||
collectionId: string;
|
||||
};
|
||||
|
||||
export type getTrainingDetailResponse = {
|
||||
trainingType: DatasetCollectionDataProcessModeEnum;
|
||||
advancedTraining: {
|
||||
customPdfParse: boolean;
|
||||
imageIndex: boolean;
|
||||
autoIndexes: boolean;
|
||||
};
|
||||
queuedCounts: Record<TrainingModeEnum, number>;
|
||||
trainingCounts: Record<TrainingModeEnum, number>;
|
||||
errorCounts: Record<TrainingModeEnum, number>;
|
||||
trainedCount: number;
|
||||
};
|
||||
|
||||
const defaultCounts: Record<TrainingModeEnum, number> = {
|
||||
qa: 0,
|
||||
chunk: 0,
|
||||
image: 0,
|
||||
auto: 0
|
||||
};
|
||||
|
||||
async function handler(
|
||||
req: ApiRequestProps<{}, getTrainingDetailParams>
|
||||
): Promise<getTrainingDetailResponse> {
|
||||
const { collectionId } = req.query;
|
||||
|
||||
const { collection } = await authDatasetCollection({
|
||||
req,
|
||||
authToken: true,
|
||||
collectionId: collectionId as string,
|
||||
per: ReadPermissionVal
|
||||
});
|
||||
|
||||
const match = {
|
||||
teamId: collection.teamId,
|
||||
datasetId: collection.datasetId,
|
||||
collectionId: collection._id
|
||||
};
|
||||
|
||||
// Computed global queue
|
||||
const minId = (
|
||||
await MongoDatasetTraining.findOne(
|
||||
{
|
||||
teamId: collection.teamId,
|
||||
datasetId: collection.datasetId,
|
||||
collectionId: collection._id
|
||||
},
|
||||
{ sort: { _id: 1 }, select: '_id' },
|
||||
readFromSecondary
|
||||
).lean()
|
||||
)?._id;
|
||||
|
||||
const [ququedCountData, trainingCountData, errorCountData, trainedCount] = (await Promise.all([
|
||||
minId
|
||||
? MongoDatasetTraining.aggregate(
|
||||
[
|
||||
{
|
||||
$match: {
|
||||
_id: { $lt: minId },
|
||||
retryCount: { $gt: 0 },
|
||||
lockTime: { $lt: new Date('2050/1/1') }
|
||||
}
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$mode',
|
||||
count: { $sum: 1 }
|
||||
}
|
||||
}
|
||||
],
|
||||
readFromSecondary
|
||||
)
|
||||
: Promise.resolve([]),
|
||||
MongoDatasetTraining.aggregate(
|
||||
[
|
||||
{
|
||||
$match: {
|
||||
...match,
|
||||
retryCount: { $gt: 0 },
|
||||
lockTime: { $lt: new Date('2050/1/1') }
|
||||
}
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$mode',
|
||||
count: { $sum: 1 }
|
||||
}
|
||||
}
|
||||
],
|
||||
readFromSecondary
|
||||
),
|
||||
MongoDatasetTraining.aggregate(
|
||||
[
|
||||
{
|
||||
$match: {
|
||||
...match,
|
||||
retryCount: { $lte: 0 },
|
||||
errorMsg: { $exists: true }
|
||||
}
|
||||
},
|
||||
{
|
||||
$group: {
|
||||
_id: '$mode',
|
||||
count: { $sum: 1 }
|
||||
}
|
||||
}
|
||||
],
|
||||
readFromSecondary
|
||||
),
|
||||
MongoDatasetData.countDocuments(match, readFromSecondary)
|
||||
])) as [
|
||||
{ _id: TrainingModeEnum; count: number }[],
|
||||
{ _id: TrainingModeEnum; count: number }[],
|
||||
{ _id: TrainingModeEnum; count: number }[],
|
||||
number
|
||||
];
|
||||
|
||||
const queuedCounts = ququedCountData.reduce(
|
||||
(acc, item) => {
|
||||
acc[item._id] = item.count;
|
||||
return acc;
|
||||
},
|
||||
{ ...defaultCounts }
|
||||
);
|
||||
const trainingCounts = trainingCountData.reduce(
|
||||
(acc, item) => {
|
||||
acc[item._id] = item.count;
|
||||
return acc;
|
||||
},
|
||||
{ ...defaultCounts }
|
||||
);
|
||||
const errorCounts = errorCountData.reduce(
|
||||
(acc, item) => {
|
||||
acc[item._id] = item.count;
|
||||
return acc;
|
||||
},
|
||||
{ ...defaultCounts }
|
||||
);
|
||||
|
||||
return {
|
||||
trainingType: collection.trainingType,
|
||||
advancedTraining: {
|
||||
customPdfParse: !!collection.customPdfParse,
|
||||
imageIndex: !!collection.imageIndex,
|
||||
autoIndexes: !!collection.autoIndexes
|
||||
},
|
||||
|
||||
queuedCounts,
|
||||
trainingCounts,
|
||||
errorCounts,
|
||||
trainedCount
|
||||
};
|
||||
}
|
||||
|
||||
export default NextAPI(handler);
|
||||
@@ -9,6 +9,8 @@ import { OwnerPermissionVal } from '@fastgpt/global/support/permission/constant'
|
||||
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
|
||||
import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema';
|
||||
import { removeImageByPath } from '@fastgpt/service/common/file/image/controller';
|
||||
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { removeWebsiteSyncJobScheduler } from '@fastgpt/service/core/dataset/websiteSync';
|
||||
|
||||
async function handler(req: NextApiRequest) {
|
||||
const { id: datasetId } = req.query as {
|
||||
@@ -40,6 +42,13 @@ async function handler(req: NextApiRequest) {
|
||||
datasetId: { $in: datasetIds }
|
||||
});
|
||||
|
||||
await Promise.all(
|
||||
datasets.map((dataset) => {
|
||||
if (dataset.type === DatasetTypeEnum.websiteDataset)
|
||||
return removeWebsiteSyncJobScheduler(String(dataset._id));
|
||||
})
|
||||
);
|
||||
|
||||
// delete all dataset.data and pg data
|
||||
await mongoSessionRun(async (session) => {
|
||||
// delete dataset data
|
||||
|
||||
@@ -5,6 +5,8 @@ import { NextAPI } from '@/service/middleware/entry';
|
||||
import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
|
||||
import { getWebsiteSyncDatasetStatus } from '@fastgpt/service/core/dataset/websiteSync';
|
||||
import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
|
||||
type Query = {
|
||||
id: string;
|
||||
@@ -28,8 +30,21 @@ async function handler(req: ApiRequestProps<Query>): Promise<DatasetItemType> {
|
||||
per: ReadPermissionVal
|
||||
});
|
||||
|
||||
const { status, errorMsg } = await (async () => {
|
||||
if (dataset.type === DatasetTypeEnum.websiteDataset) {
|
||||
return await getWebsiteSyncDatasetStatus(datasetId);
|
||||
}
|
||||
|
||||
return {
|
||||
status: DatasetStatusEnum.active,
|
||||
errorMsg: undefined
|
||||
};
|
||||
})();
|
||||
|
||||
return {
|
||||
...dataset,
|
||||
status,
|
||||
errorMsg,
|
||||
apiServer: dataset.apiServer
|
||||
? {
|
||||
baseUrl: dataset.apiServer.baseUrl,
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
import { ManagePermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
|
||||
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
|
||||
export type deleteTrainingDataBody = {
|
||||
datasetId: string;
|
||||
collectionId: string;
|
||||
dataId: string;
|
||||
};
|
||||
|
||||
export type deleteTrainingDataQuery = {};
|
||||
|
||||
export type deleteTrainingDataResponse = {};
|
||||
|
||||
async function handler(
|
||||
req: ApiRequestProps<deleteTrainingDataBody, deleteTrainingDataQuery>
|
||||
): Promise<deleteTrainingDataResponse> {
|
||||
const { datasetId, collectionId, dataId } = req.body;
|
||||
|
||||
const { teamId } = await authDatasetCollection({
|
||||
req,
|
||||
authToken: true,
|
||||
authApiKey: true,
|
||||
collectionId,
|
||||
per: ManagePermissionVal
|
||||
});
|
||||
|
||||
await MongoDatasetTraining.deleteOne({
|
||||
teamId,
|
||||
datasetId,
|
||||
_id: dataId
|
||||
});
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
export default NextAPI(handler);
|
||||
@@ -0,0 +1,52 @@
|
||||
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
|
||||
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
|
||||
export type getTrainingDataDetailQuery = {};
|
||||
|
||||
export type getTrainingDataDetailBody = {
|
||||
datasetId: string;
|
||||
collectionId: string;
|
||||
dataId: string;
|
||||
};
|
||||
|
||||
export type getTrainingDataDetailResponse =
|
||||
| {
|
||||
_id: string;
|
||||
datasetId: string;
|
||||
mode: string;
|
||||
q: string;
|
||||
a: string;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
async function handler(
|
||||
req: ApiRequestProps<getTrainingDataDetailBody, getTrainingDataDetailQuery>
|
||||
): Promise<getTrainingDataDetailResponse> {
|
||||
const { datasetId, collectionId, dataId } = req.body;
|
||||
|
||||
const { teamId } = await authDatasetCollection({
|
||||
req,
|
||||
authToken: true,
|
||||
collectionId,
|
||||
per: ReadPermissionVal
|
||||
});
|
||||
|
||||
const data = await MongoDatasetTraining.findOne({ teamId, datasetId, _id: dataId }).lean();
|
||||
|
||||
if (!data) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return {
|
||||
_id: data._id,
|
||||
datasetId: data.datasetId,
|
||||
mode: data.mode,
|
||||
q: data.q,
|
||||
a: data.a
|
||||
};
|
||||
}
|
||||
|
||||
export default NextAPI(handler);
|
||||
@@ -0,0 +1,51 @@
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
|
||||
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
|
||||
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
|
||||
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
|
||||
|
||||
export type getTrainingErrorBody = PaginationProps<{
|
||||
collectionId: string;
|
||||
}>;
|
||||
|
||||
export type getTrainingErrorResponse = PaginationResponse<DatasetTrainingSchemaType>;
|
||||
|
||||
async function handler(req: ApiRequestProps<getTrainingErrorBody, {}>) {
|
||||
const { collectionId } = req.body;
|
||||
const { offset, pageSize } = parsePaginationRequest(req);
|
||||
|
||||
const { collection } = await authDatasetCollection({
|
||||
req,
|
||||
authToken: true,
|
||||
collectionId,
|
||||
per: ReadPermissionVal
|
||||
});
|
||||
|
||||
const match = {
|
||||
teamId: collection.teamId,
|
||||
datasetId: collection.datasetId,
|
||||
collectionId: collection._id,
|
||||
errorMsg: { $exists: true }
|
||||
};
|
||||
|
||||
const [errorList, total] = await Promise.all([
|
||||
MongoDatasetTraining.find(match, undefined, {
|
||||
...readFromSecondary
|
||||
})
|
||||
.skip(offset)
|
||||
.limit(pageSize)
|
||||
.lean(),
|
||||
MongoDatasetTraining.countDocuments(match, { ...readFromSecondary })
|
||||
]);
|
||||
|
||||
return {
|
||||
list: errorList,
|
||||
total
|
||||
};
|
||||
}
|
||||
|
||||
export default NextAPI(handler);
|
||||
@@ -0,0 +1,59 @@
|
||||
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
|
||||
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
import { addMinutes } from 'date-fns';
|
||||
|
||||
export type updateTrainingDataBody = {
|
||||
datasetId: string;
|
||||
collectionId: string;
|
||||
dataId: string;
|
||||
q?: string;
|
||||
a?: string;
|
||||
chunkIndex?: number;
|
||||
};
|
||||
|
||||
export type updateTrainingDataQuery = {};
|
||||
|
||||
export type updateTrainingDataResponse = {};
|
||||
|
||||
async function handler(
|
||||
req: ApiRequestProps<updateTrainingDataBody, updateTrainingDataQuery>
|
||||
): Promise<updateTrainingDataResponse> {
|
||||
const { datasetId, collectionId, dataId, q, a, chunkIndex } = req.body;
|
||||
|
||||
const { teamId } = await authDatasetCollection({
|
||||
req,
|
||||
authToken: true,
|
||||
authApiKey: true,
|
||||
collectionId,
|
||||
per: WritePermissionVal
|
||||
});
|
||||
|
||||
const data = await MongoDatasetTraining.findOne({ teamId, datasetId, _id: dataId });
|
||||
|
||||
if (!data) {
|
||||
return Promise.reject('data not found');
|
||||
}
|
||||
|
||||
await MongoDatasetTraining.updateOne(
|
||||
{
|
||||
teamId,
|
||||
datasetId,
|
||||
_id: dataId
|
||||
},
|
||||
{
|
||||
$unset: { errorMsg: '' },
|
||||
retryCount: 3,
|
||||
...(q !== undefined && { q }),
|
||||
...(a !== undefined && { a }),
|
||||
...(chunkIndex !== undefined && { chunkIndex }),
|
||||
lockTime: addMinutes(new Date(), -10)
|
||||
}
|
||||
);
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
export default NextAPI(handler);
|
||||
@@ -30,6 +30,13 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection
|
||||
import { addDays } from 'date-fns';
|
||||
import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller';
|
||||
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
|
||||
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import {
|
||||
removeWebsiteSyncJobScheduler,
|
||||
upsertWebsiteSyncJobScheduler
|
||||
} from '@fastgpt/service/core/dataset/websiteSync';
|
||||
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller';
|
||||
import { isEqual } from 'lodash';
|
||||
|
||||
export type DatasetUpdateQuery = {};
|
||||
export type DatasetUpdateResponse = any;
|
||||
@@ -62,8 +69,8 @@ async function handler(
|
||||
apiServer,
|
||||
yuqueServer,
|
||||
feishuServer,
|
||||
status,
|
||||
autoSync
|
||||
autoSync,
|
||||
chunkSettings
|
||||
} = req.body;
|
||||
|
||||
if (!id) {
|
||||
@@ -114,6 +121,39 @@ async function handler(
|
||||
});
|
||||
|
||||
const onUpdate = async (session: ClientSession) => {
|
||||
// Website dataset update chunkSettings, need to clean up dataset
|
||||
if (
|
||||
dataset.type === DatasetTypeEnum.websiteDataset &&
|
||||
chunkSettings &&
|
||||
dataset.chunkSettings &&
|
||||
!isEqual(
|
||||
{
|
||||
imageIndex: dataset.chunkSettings.imageIndex,
|
||||
autoIndexes: dataset.chunkSettings.autoIndexes,
|
||||
trainingType: dataset.chunkSettings.trainingType,
|
||||
chunkSettingMode: dataset.chunkSettings.chunkSettingMode,
|
||||
chunkSplitMode: dataset.chunkSettings.chunkSplitMode,
|
||||
chunkSize: dataset.chunkSettings.chunkSize,
|
||||
chunkSplitter: dataset.chunkSettings.chunkSplitter,
|
||||
indexSize: dataset.chunkSettings.indexSize,
|
||||
qaPrompt: dataset.chunkSettings.qaPrompt
|
||||
},
|
||||
{
|
||||
imageIndex: chunkSettings.imageIndex,
|
||||
autoIndexes: chunkSettings.autoIndexes,
|
||||
trainingType: chunkSettings.trainingType,
|
||||
chunkSettingMode: chunkSettings.chunkSettingMode,
|
||||
chunkSplitMode: chunkSettings.chunkSplitMode,
|
||||
chunkSize: chunkSettings.chunkSize,
|
||||
chunkSplitter: chunkSettings.chunkSplitter,
|
||||
indexSize: chunkSettings.indexSize,
|
||||
qaPrompt: chunkSettings.qaPrompt
|
||||
}
|
||||
)
|
||||
) {
|
||||
await delDatasetRelevantData({ datasets: [dataset], session });
|
||||
}
|
||||
|
||||
await MongoDataset.findByIdAndUpdate(
|
||||
id,
|
||||
{
|
||||
@@ -123,7 +163,7 @@ async function handler(
|
||||
...(agentModel && { agentModel }),
|
||||
...(vlmModel && { vlmModel }),
|
||||
...(websiteConfig && { websiteConfig }),
|
||||
...(status && { status }),
|
||||
...(chunkSettings && { chunkSettings }),
|
||||
...(intro !== undefined && { intro }),
|
||||
...(externalReadUrl !== undefined && { externalReadUrl }),
|
||||
...(!!apiServer?.baseUrl && { 'apiServer.baseUrl': apiServer.baseUrl }),
|
||||
@@ -143,8 +183,7 @@ async function handler(
|
||||
{ session }
|
||||
);
|
||||
await updateSyncSchedule({
|
||||
teamId: dataset.teamId,
|
||||
datasetId: dataset._id,
|
||||
dataset,
|
||||
autoSync,
|
||||
session
|
||||
});
|
||||
@@ -221,45 +260,54 @@ const updateTraining = async ({
|
||||
};
|
||||
|
||||
const updateSyncSchedule = async ({
|
||||
teamId,
|
||||
datasetId,
|
||||
dataset,
|
||||
autoSync,
|
||||
session
|
||||
}: {
|
||||
teamId: string;
|
||||
datasetId: string;
|
||||
dataset: DatasetSchemaType;
|
||||
autoSync?: boolean;
|
||||
session: ClientSession;
|
||||
}) => {
|
||||
if (typeof autoSync !== 'boolean') return;
|
||||
|
||||
// Update all collection nextSyncTime
|
||||
if (autoSync) {
|
||||
await MongoDatasetCollection.updateMany(
|
||||
{
|
||||
teamId,
|
||||
datasetId,
|
||||
type: { $in: [DatasetCollectionTypeEnum.apiFile, DatasetCollectionTypeEnum.link] }
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextSyncTime: addDays(new Date(), 1)
|
||||
}
|
||||
},
|
||||
{ session }
|
||||
);
|
||||
if (dataset.type === DatasetTypeEnum.websiteDataset) {
|
||||
if (autoSync) {
|
||||
// upsert Job Scheduler
|
||||
upsertWebsiteSyncJobScheduler({ datasetId: String(dataset._id) });
|
||||
} else {
|
||||
// remove Job Scheduler
|
||||
removeWebsiteSyncJobScheduler(String(dataset._id));
|
||||
}
|
||||
} else {
|
||||
await MongoDatasetCollection.updateMany(
|
||||
{
|
||||
teamId,
|
||||
datasetId
|
||||
},
|
||||
{
|
||||
$unset: {
|
||||
nextSyncTime: 1
|
||||
}
|
||||
},
|
||||
{ session }
|
||||
);
|
||||
// Other dataset, update the collection sync
|
||||
if (autoSync) {
|
||||
await MongoDatasetCollection.updateMany(
|
||||
{
|
||||
teamId: dataset.teamId,
|
||||
datasetId: dataset._id,
|
||||
type: { $in: [DatasetCollectionTypeEnum.apiFile, DatasetCollectionTypeEnum.link] }
|
||||
},
|
||||
{
|
||||
$set: {
|
||||
nextSyncTime: addDays(new Date(), 1)
|
||||
}
|
||||
},
|
||||
{ session }
|
||||
);
|
||||
} else {
|
||||
await MongoDatasetCollection.updateMany(
|
||||
{
|
||||
teamId: dataset.teamId,
|
||||
datasetId: dataset._id
|
||||
},
|
||||
{
|
||||
$unset: {
|
||||
nextSyncTime: 1
|
||||
}
|
||||
},
|
||||
{ session }
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { request } from 'https';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
const { path = [], ...query } = req.query as any;
|
||||
|
||||
const queryStr = new URLSearchParams(query).toString();
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { request } from 'http';
|
||||
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
const { path = [], ...query } = req.query as any;
|
||||
const requestPath = `/api/${path?.join('/')}?${new URLSearchParams(query).toString()}`;
|
||||
|
||||
|
||||
@@ -2,12 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { MongoUser } from '@fastgpt/service/support/user/schema';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
const { oldPsw, newPsw } = req.body as { oldPsw: string; newPsw: string };
|
||||
|
||||
if (!oldPsw || !newPsw) {
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
const { size } = req.query as {
|
||||
size: string;
|
||||
};
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { checkWebSyncLimit } from '@fastgpt/service/support/user/utils';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
|
||||
// 凭证校验
|
||||
const { teamId } = await authCert({ req, authToken: true });
|
||||
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { jsonRes } from '@fastgpt/service/common/response';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import { readMongoImg } from '@fastgpt/service/common/file/image/controller';
|
||||
|
||||
// get the models available to the system
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
const { id } = req.query as { id: string };
|
||||
|
||||
const { binary, mime } = await readMongoImg({ id });
|
||||
|
||||
@@ -59,7 +59,6 @@ import { getWorkflowResponseWrite } from '@fastgpt/service/core/workflow/dispatc
|
||||
import { WORKFLOW_MAX_RUN_TIMES } from '@fastgpt/service/core/workflow/constants';
|
||||
import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
|
||||
import { ExternalProviderType } from '@fastgpt/global/core/workflow/runtime/type';
|
||||
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
|
||||
|
||||
type FastGptWebChatProps = {
|
||||
chatId?: string; // undefined: get histories from messages, '': new chat, 'xxxxx': get histories from db
|
||||
|
||||
641
projects/app/src/pages/api/v2/chat/completions.ts
Normal file
641
projects/app/src/pages/api/v2/chat/completions.ts
Normal file
@@ -0,0 +1,641 @@
|
||||
import type { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { authApp } from '@fastgpt/service/support/permission/app/auth';
|
||||
import { authCert } from '@fastgpt/service/support/permission/auth/common';
|
||||
import { sseErrRes, jsonRes } from '@fastgpt/service/common/response';
|
||||
import { addLog } from '@fastgpt/service/common/system/log';
|
||||
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
|
||||
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
|
||||
import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
|
||||
import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d';
|
||||
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d';
|
||||
import {
|
||||
getWorkflowEntryNodeIds,
|
||||
getMaxHistoryLimitFromNodes,
|
||||
initWorkflowEdgeStatus,
|
||||
storeNodes2RuntimeNodes,
|
||||
textAdaptGptResponse,
|
||||
getLastInteractiveValue
|
||||
} from '@fastgpt/global/core/workflow/runtime/utils';
|
||||
import { GPTMessages2Chats, chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
|
||||
import { getChatItems } from '@fastgpt/service/core/chat/controller';
|
||||
import { saveChat, updateInteractiveChat } from '@fastgpt/service/core/chat/saveChat';
|
||||
import { responseWrite } from '@fastgpt/service/common/response';
|
||||
import { createChatUsage } from '@fastgpt/service/support/wallet/usage/controller';
|
||||
import { authOutLinkChatStart } from '@/service/support/permission/auth/outLink';
|
||||
import { pushResult2Remote, addOutLinkUsage } from '@fastgpt/service/support/outLink/tools';
|
||||
import requestIp from 'request-ip';
|
||||
import { getUsageSourceByAuthType } from '@fastgpt/global/support/wallet/usage/tools';
|
||||
import { authTeamSpaceToken } from '@/service/support/permission/auth/team';
|
||||
import {
|
||||
concatHistories,
|
||||
filterPublicNodeResponseData,
|
||||
getChatTitleFromChatMessage,
|
||||
removeEmptyUserInput
|
||||
} from '@fastgpt/global/core/chat/utils';
|
||||
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
|
||||
import { getUserChatInfoAndAuthTeamPoints } from '@fastgpt/service/support/permission/auth/team';
|
||||
import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
|
||||
import { MongoApp } from '@fastgpt/service/core/app/schema';
|
||||
import { AppSchema } from '@fastgpt/global/core/app/type';
|
||||
import { AuthOutLinkChatProps } from '@fastgpt/global/support/outLink/api';
|
||||
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
|
||||
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
|
||||
import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
|
||||
import { AIChatItemType, UserChatItemType } from '@fastgpt/global/core/chat/type';
|
||||
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
|
||||
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { getAppLatestVersion } from '@fastgpt/service/core/app/version/controller';
|
||||
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
|
||||
import {
|
||||
getPluginRunUserQuery,
|
||||
updatePluginInputByVariables
|
||||
} from '@fastgpt/global/core/workflow/utils';
|
||||
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
||||
import { getSystemTime } from '@fastgpt/global/common/time/timezone';
|
||||
import { rewriteNodeOutputByHistories } from '@fastgpt/global/core/workflow/runtime/utils';
|
||||
import { getWorkflowResponseWrite } from '@fastgpt/service/core/workflow/dispatch/utils';
|
||||
import { WORKFLOW_MAX_RUN_TIMES } from '@fastgpt/service/core/workflow/constants';
|
||||
import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
|
||||
import { ExternalProviderType } from '@fastgpt/global/core/workflow/runtime/type';
|
||||
|
||||
type FastGptWebChatProps = {
|
||||
chatId?: string; // undefined: get histories from messages, '': new chat, 'xxxxx': get histories from db
|
||||
appId?: string;
|
||||
customUid?: string; // non-undefined: will be the priority provider for the logger.
|
||||
metadata?: Record<string, any>;
|
||||
};
|
||||
|
||||
export type Props = ChatCompletionCreateParams &
|
||||
FastGptWebChatProps &
|
||||
OutLinkChatAuthProps & {
|
||||
messages: ChatCompletionMessageParam[];
|
||||
responseChatItemId?: string;
|
||||
stream?: boolean;
|
||||
detail?: boolean;
|
||||
variables: Record<string, any>; // Global variables or plugin inputs
|
||||
};
|
||||
|
||||
type AuthResponseType = {
|
||||
teamId: string;
|
||||
tmbId: string;
|
||||
timezone: string;
|
||||
externalProvider: ExternalProviderType;
|
||||
app: AppSchema;
|
||||
responseDetail?: boolean;
|
||||
showNodeStatus?: boolean;
|
||||
authType: `${AuthUserTypeEnum}`;
|
||||
apikey?: string;
|
||||
responseAllData: boolean;
|
||||
outLinkUserId?: string;
|
||||
sourceName?: string;
|
||||
};
|
||||
|
||||
async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
res.on('close', () => {
|
||||
res.end();
|
||||
});
|
||||
res.on('error', () => {
|
||||
console.log('error: ', 'request error');
|
||||
res.end();
|
||||
});
|
||||
|
||||
let {
|
||||
chatId,
|
||||
appId,
|
||||
customUid,
|
||||
// share chat
|
||||
shareId,
|
||||
outLinkUid,
|
||||
// team chat
|
||||
teamId: spaceTeamId,
|
||||
teamToken,
|
||||
|
||||
stream = false,
|
||||
detail = false,
|
||||
messages = [],
|
||||
variables = {},
|
||||
responseChatItemId = getNanoid(),
|
||||
metadata
|
||||
} = req.body as Props;
|
||||
|
||||
const originIp = requestIp.getClientIp(req);
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
try {
|
||||
if (!Array.isArray(messages)) {
|
||||
throw new Error('messages is not array');
|
||||
}
|
||||
|
||||
/*
|
||||
Web params: chatId + [Human]
|
||||
API params: chatId + [Human]
|
||||
API params: [histories, Human]
|
||||
*/
|
||||
const chatMessages = GPTMessages2Chats(messages);
|
||||
|
||||
// Computed start hook params
|
||||
const startHookText = (() => {
|
||||
// Chat
|
||||
const userQuestion = chatMessages[chatMessages.length - 1] as UserChatItemType | undefined;
|
||||
if (userQuestion) return chatValue2RuntimePrompt(userQuestion.value).text;
|
||||
|
||||
// plugin
|
||||
return JSON.stringify(variables);
|
||||
})();
|
||||
|
||||
/*
|
||||
1. auth app permission
|
||||
2. auth balance
|
||||
3. get app
|
||||
4. parse outLink token
|
||||
*/
|
||||
const {
|
||||
teamId,
|
||||
tmbId,
|
||||
timezone,
|
||||
externalProvider,
|
||||
app,
|
||||
responseDetail,
|
||||
authType,
|
||||
sourceName,
|
||||
apikey,
|
||||
responseAllData,
|
||||
outLinkUserId = customUid,
|
||||
showNodeStatus
|
||||
} = await (async () => {
|
||||
// share chat
|
||||
if (shareId && outLinkUid) {
|
||||
return authShareChat({
|
||||
shareId,
|
||||
outLinkUid,
|
||||
chatId,
|
||||
ip: originIp,
|
||||
question: startHookText
|
||||
});
|
||||
}
|
||||
// team space chat
|
||||
if (spaceTeamId && appId && teamToken) {
|
||||
return authTeamSpaceChat({
|
||||
teamId: spaceTeamId,
|
||||
teamToken,
|
||||
appId,
|
||||
chatId
|
||||
});
|
||||
}
|
||||
|
||||
/* parse req: api or token */
|
||||
return authHeaderRequest({
|
||||
req,
|
||||
appId,
|
||||
chatId
|
||||
});
|
||||
})();
|
||||
const isPlugin = app.type === AppTypeEnum.plugin;
|
||||
|
||||
// Check message type
|
||||
if (isPlugin) {
|
||||
detail = true;
|
||||
} else {
|
||||
if (messages.length === 0) {
|
||||
throw new Error('messages is empty');
|
||||
}
|
||||
}
|
||||
|
||||
// Get obj=Human history
|
||||
const userQuestion: UserChatItemType = (() => {
|
||||
if (isPlugin) {
|
||||
return getPluginRunUserQuery({
|
||||
pluginInputs: getPluginInputsFromStoreNodes(app.modules),
|
||||
variables,
|
||||
files: variables.files
|
||||
});
|
||||
}
|
||||
|
||||
const latestHumanChat = chatMessages.pop() as UserChatItemType | undefined;
|
||||
if (!latestHumanChat) {
|
||||
throw new Error('User question is empty');
|
||||
}
|
||||
return latestHumanChat;
|
||||
})();
|
||||
|
||||
// Get and concat history;
|
||||
const limit = getMaxHistoryLimitFromNodes(app.modules);
|
||||
const [{ histories }, { nodes, edges, chatConfig }, chatDetail] = await Promise.all([
|
||||
getChatItems({
|
||||
appId: app._id,
|
||||
chatId,
|
||||
offset: 0,
|
||||
limit,
|
||||
field: `dataId obj value nodeOutputs`
|
||||
}),
|
||||
getAppLatestVersion(app._id, app),
|
||||
MongoChat.findOne({ appId: app._id, chatId }, 'source variableList variables')
|
||||
]);
|
||||
|
||||
// Get store variables(Api variable precedence)
|
||||
if (chatDetail?.variables) {
|
||||
variables = {
|
||||
...chatDetail.variables,
|
||||
...variables
|
||||
};
|
||||
}
|
||||
|
||||
// Get chat histories
|
||||
const newHistories = concatHistories(histories, chatMessages);
|
||||
|
||||
// Get runtimeNodes
|
||||
let runtimeNodes = storeNodes2RuntimeNodes(nodes, getWorkflowEntryNodeIds(nodes, newHistories));
|
||||
if (isPlugin) {
|
||||
// Assign values to runtimeNodes using variables
|
||||
runtimeNodes = updatePluginInputByVariables(runtimeNodes, variables);
|
||||
// Plugin runtime does not need global variables(It has been injected into the pluginInputNode)
|
||||
variables = {};
|
||||
}
|
||||
runtimeNodes = rewriteNodeOutputByHistories(newHistories, runtimeNodes);
|
||||
|
||||
const workflowResponseWrite = getWorkflowResponseWrite({
|
||||
res,
|
||||
detail,
|
||||
streamResponse: stream,
|
||||
id: chatId,
|
||||
showNodeStatus
|
||||
});
|
||||
|
||||
/* start flow controller */
|
||||
const { flowResponses, flowUsages, assistantResponses, newVariables } = await (async () => {
|
||||
if (app.version === 'v2') {
|
||||
return dispatchWorkFlow({
|
||||
res,
|
||||
requestOrigin: req.headers.origin,
|
||||
mode: 'chat',
|
||||
timezone,
|
||||
externalProvider,
|
||||
|
||||
runningAppInfo: {
|
||||
id: String(app._id),
|
||||
teamId: String(app.teamId),
|
||||
tmbId: String(app.tmbId)
|
||||
},
|
||||
runningUserInfo: {
|
||||
teamId,
|
||||
tmbId
|
||||
},
|
||||
uid: String(outLinkUserId || tmbId),
|
||||
|
||||
chatId,
|
||||
responseChatItemId,
|
||||
runtimeNodes,
|
||||
runtimeEdges: initWorkflowEdgeStatus(edges, newHistories),
|
||||
variables,
|
||||
query: removeEmptyUserInput(userQuestion.value),
|
||||
chatConfig,
|
||||
histories: newHistories,
|
||||
stream,
|
||||
maxRunTimes: WORKFLOW_MAX_RUN_TIMES,
|
||||
workflowStreamResponse: workflowResponseWrite,
|
||||
version: 'v2'
|
||||
});
|
||||
}
|
||||
return Promise.reject('您的工作流版本过低,请重新发布一次');
|
||||
})();
|
||||
|
||||
// save chat
|
||||
const isOwnerUse = !shareId && !spaceTeamId && String(tmbId) === String(app.tmbId);
|
||||
const source = (() => {
|
||||
if (shareId) {
|
||||
return ChatSourceEnum.share;
|
||||
}
|
||||
if (authType === 'apikey') {
|
||||
return ChatSourceEnum.api;
|
||||
}
|
||||
if (spaceTeamId) {
|
||||
return ChatSourceEnum.team;
|
||||
}
|
||||
return ChatSourceEnum.online;
|
||||
})();
|
||||
|
||||
const isInteractiveRequest = !!getLastInteractiveValue(histories);
|
||||
const { text: userInteractiveVal } = chatValue2RuntimePrompt(userQuestion.value);
|
||||
|
||||
const newTitle = isPlugin
|
||||
? variables.cTime ?? getSystemTime(timezone)
|
||||
: getChatTitleFromChatMessage(userQuestion);
|
||||
|
||||
const aiResponse: AIChatItemType & { dataId?: string } = {
|
||||
dataId: responseChatItemId,
|
||||
obj: ChatRoleEnum.AI,
|
||||
value: assistantResponses,
|
||||
[DispatchNodeResponseKeyEnum.nodeResponse]: flowResponses
|
||||
};
|
||||
|
||||
const saveChatId = chatId || getNanoid(24);
|
||||
if (isInteractiveRequest) {
|
||||
await updateInteractiveChat({
|
||||
chatId: saveChatId,
|
||||
appId: app._id,
|
||||
userInteractiveVal,
|
||||
aiResponse,
|
||||
newVariables
|
||||
});
|
||||
} else {
|
||||
await saveChat({
|
||||
chatId: saveChatId,
|
||||
appId: app._id,
|
||||
teamId,
|
||||
tmbId: tmbId,
|
||||
nodes,
|
||||
appChatConfig: chatConfig,
|
||||
variables: newVariables,
|
||||
isUpdateUseTime: isOwnerUse && source === ChatSourceEnum.online, // owner update use time
|
||||
newTitle,
|
||||
shareId,
|
||||
outLinkUid: outLinkUserId,
|
||||
source,
|
||||
sourceName: sourceName || '',
|
||||
content: [userQuestion, aiResponse],
|
||||
metadata: {
|
||||
originIp,
|
||||
...metadata
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
addLog.info(`completions running time: ${(Date.now() - startTime) / 1000}s`);
|
||||
|
||||
/* select fe response field */
|
||||
const feResponseData = responseAllData
|
||||
? flowResponses
|
||||
: filterPublicNodeResponseData({ flowResponses, responseDetail });
|
||||
|
||||
if (stream) {
|
||||
workflowResponseWrite({
|
||||
event: SseResponseEventEnum.answer,
|
||||
data: textAdaptGptResponse({
|
||||
text: null,
|
||||
finish_reason: 'stop'
|
||||
})
|
||||
});
|
||||
responseWrite({
|
||||
res,
|
||||
event: detail ? SseResponseEventEnum.answer : undefined,
|
||||
data: '[DONE]'
|
||||
});
|
||||
|
||||
res.end();
|
||||
} else {
|
||||
const responseContent = (() => {
|
||||
if (assistantResponses.length === 0) return '';
|
||||
if (assistantResponses.length === 1 && assistantResponses[0].text?.content)
|
||||
return assistantResponses[0].text?.content;
|
||||
|
||||
if (!detail) {
|
||||
return assistantResponses
|
||||
.map((item) => item?.text?.content)
|
||||
.filter(Boolean)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
return assistantResponses;
|
||||
})();
|
||||
const error = flowResponses[flowResponses.length - 1]?.error;
|
||||
|
||||
res.json({
|
||||
...(detail ? { responseData: feResponseData, newVariables } : {}),
|
||||
error,
|
||||
id: chatId || '',
|
||||
model: '',
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 1 },
|
||||
choices: [
|
||||
{
|
||||
message: { role: 'assistant', content: responseContent },
|
||||
finish_reason: 'stop',
|
||||
index: 0
|
||||
}
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
// add record
|
||||
const { totalPoints } = createChatUsage({
|
||||
appName: app.name,
|
||||
appId: app._id,
|
||||
teamId,
|
||||
tmbId: tmbId,
|
||||
source: getUsageSourceByAuthType({ shareId, authType }),
|
||||
flowUsages
|
||||
});
|
||||
|
||||
if (shareId) {
|
||||
pushResult2Remote({ outLinkUid, shareId, appName: app.name, flowResponses });
|
||||
addOutLinkUsage({
|
||||
shareId,
|
||||
totalPoints
|
||||
});
|
||||
}
|
||||
if (apikey) {
|
||||
updateApiKeyUsage({
|
||||
apikey,
|
||||
totalPoints
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
if (stream) {
|
||||
sseErrRes(res, err);
|
||||
res.end();
|
||||
} else {
|
||||
jsonRes(res, {
|
||||
code: 500,
|
||||
error: err
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
export default NextAPI(handler);
|
||||
|
||||
const authShareChat = async ({
|
||||
chatId,
|
||||
...data
|
||||
}: AuthOutLinkChatProps & {
|
||||
shareId: string;
|
||||
chatId?: string;
|
||||
}): Promise<AuthResponseType> => {
|
||||
const {
|
||||
teamId,
|
||||
tmbId,
|
||||
timezone,
|
||||
externalProvider,
|
||||
appId,
|
||||
authType,
|
||||
responseDetail,
|
||||
showNodeStatus,
|
||||
uid,
|
||||
sourceName
|
||||
} = await authOutLinkChatStart(data);
|
||||
const app = await MongoApp.findById(appId).lean();
|
||||
|
||||
if (!app) {
|
||||
return Promise.reject('app is empty');
|
||||
}
|
||||
|
||||
// get chat
|
||||
const chat = await MongoChat.findOne({ appId, chatId }).lean();
|
||||
if (chat && (chat.shareId !== data.shareId || chat.outLinkUid !== uid)) {
|
||||
return Promise.reject(ChatErrEnum.unAuthChat);
|
||||
}
|
||||
|
||||
return {
|
||||
sourceName,
|
||||
teamId,
|
||||
tmbId,
|
||||
app,
|
||||
timezone,
|
||||
externalProvider,
|
||||
apikey: '',
|
||||
authType,
|
||||
responseAllData: false,
|
||||
responseDetail,
|
||||
outLinkUserId: uid,
|
||||
showNodeStatus
|
||||
};
|
||||
};
|
||||
const authTeamSpaceChat = async ({
|
||||
appId,
|
||||
teamId,
|
||||
teamToken,
|
||||
chatId
|
||||
}: {
|
||||
appId: string;
|
||||
teamId: string;
|
||||
teamToken: string;
|
||||
chatId?: string;
|
||||
}): Promise<AuthResponseType> => {
|
||||
const { uid } = await authTeamSpaceToken({
|
||||
teamId,
|
||||
teamToken
|
||||
});
|
||||
|
||||
const app = await MongoApp.findById(appId).lean();
|
||||
if (!app) {
|
||||
return Promise.reject('app is empty');
|
||||
}
|
||||
|
||||
const [chat, { timezone, externalProvider }] = await Promise.all([
|
||||
MongoChat.findOne({ appId, chatId }).lean(),
|
||||
getUserChatInfoAndAuthTeamPoints(app.tmbId)
|
||||
]);
|
||||
|
||||
if (chat && (String(chat.teamId) !== teamId || chat.outLinkUid !== uid)) {
|
||||
return Promise.reject(ChatErrEnum.unAuthChat);
|
||||
}
|
||||
|
||||
return {
|
||||
teamId,
|
||||
tmbId: app.tmbId,
|
||||
app,
|
||||
timezone,
|
||||
externalProvider,
|
||||
authType: AuthUserTypeEnum.outLink,
|
||||
apikey: '',
|
||||
responseAllData: false,
|
||||
responseDetail: true,
|
||||
outLinkUserId: uid
|
||||
};
|
||||
};
|
||||
const authHeaderRequest = async ({
|
||||
req,
|
||||
appId,
|
||||
chatId
|
||||
}: {
|
||||
req: NextApiRequest;
|
||||
appId?: string;
|
||||
chatId?: string;
|
||||
}): Promise<AuthResponseType> => {
|
||||
const {
|
||||
appId: apiKeyAppId,
|
||||
teamId,
|
||||
tmbId,
|
||||
authType,
|
||||
sourceName,
|
||||
apikey
|
||||
} = await authCert({
|
||||
req,
|
||||
authToken: true,
|
||||
authApiKey: true
|
||||
});
|
||||
|
||||
const { app } = await (async () => {
|
||||
if (authType === AuthUserTypeEnum.apikey) {
|
||||
const currentAppId = apiKeyAppId || appId;
|
||||
if (!currentAppId) {
|
||||
return Promise.reject(
|
||||
'Key is error. You need to use the app key rather than the account key.'
|
||||
);
|
||||
}
|
||||
const app = await MongoApp.findById(currentAppId);
|
||||
|
||||
if (!app) {
|
||||
return Promise.reject('app is empty');
|
||||
}
|
||||
|
||||
appId = String(app._id);
|
||||
|
||||
return {
|
||||
app
|
||||
};
|
||||
} else {
|
||||
// token_auth
|
||||
if (!appId) {
|
||||
return Promise.reject('appId is empty');
|
||||
}
|
||||
const { app } = await authApp({
|
||||
req,
|
||||
authToken: true,
|
||||
appId,
|
||||
per: ReadPermissionVal
|
||||
});
|
||||
|
||||
return {
|
||||
app
|
||||
};
|
||||
}
|
||||
})();
|
||||
|
||||
const [{ timezone, externalProvider }, chat] = await Promise.all([
|
||||
getUserChatInfoAndAuthTeamPoints(tmbId),
|
||||
MongoChat.findOne({ appId, chatId }).lean()
|
||||
]);
|
||||
|
||||
if (
|
||||
chat &&
|
||||
(String(chat.teamId) !== teamId ||
|
||||
// There's no need to distinguish who created it if it's apiKey auth
|
||||
(authType === AuthUserTypeEnum.token && String(chat.tmbId) !== tmbId))
|
||||
) {
|
||||
return Promise.reject(ChatErrEnum.unAuthChat);
|
||||
}
|
||||
|
||||
return {
|
||||
teamId,
|
||||
tmbId,
|
||||
timezone,
|
||||
externalProvider,
|
||||
app,
|
||||
apikey,
|
||||
authType,
|
||||
sourceName,
|
||||
responseAllData: true,
|
||||
responseDetail: true
|
||||
};
|
||||
};
|
||||
|
||||
export const config = {
|
||||
api: {
|
||||
bodyParser: {
|
||||
sizeLimit: '20mb'
|
||||
},
|
||||
responseLimit: '20mb'
|
||||
}
|
||||
};
|
||||
@@ -115,7 +115,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
|
||||
}: StartChatFnProps) => {
|
||||
// Just send a user prompt
|
||||
const histories = messages.slice(-1);
|
||||
const { responseText, responseData } = await streamFetch({
|
||||
const { responseText } = await streamFetch({
|
||||
data: {
|
||||
messages: histories,
|
||||
variables,
|
||||
@@ -137,7 +137,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
|
||||
title: newTitle
|
||||
}));
|
||||
|
||||
return { responseText, responseData, isNewChat: forbidLoadChat.current };
|
||||
return { responseText, isNewChat: forbidLoadChat.current };
|
||||
},
|
||||
[appId, chatId, onUpdateHistoryTitle, setChatBoxData, forbidLoadChat]
|
||||
);
|
||||
@@ -174,13 +174,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
|
||||
)}
|
||||
|
||||
{(!quoteData || isPc) && (
|
||||
<PageContainer
|
||||
isLoading={loading}
|
||||
flex={'1 0 0'}
|
||||
w={0}
|
||||
p={[0, '16px']}
|
||||
position={'relative'}
|
||||
>
|
||||
<PageContainer flex={'1 0 0'} w={0} p={[0, '16px']} position={'relative'}>
|
||||
<Flex h={'100%'} flexDirection={['column', 'row']}>
|
||||
{/* pc always show history. */}
|
||||
{RenderHistorySlider}
|
||||
|
||||
@@ -17,7 +17,7 @@ import { getInitOutLinkChatInfo } from '@/web/core/chat/api';
|
||||
import { getChatTitleFromChatMessage } from '@fastgpt/global/core/chat/utils';
|
||||
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
|
||||
import { addLog } from '@fastgpt/service/common/system/log';
|
||||
import { connectToDatabase } from '@/service/mongo';
|
||||
|
||||
import NextHead from '@/components/common/NextHead';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import ChatContextProvider, { ChatContext } from '@/web/core/chat/context/chatContext';
|
||||
@@ -151,7 +151,7 @@ const OutLink = (props: Props) => {
|
||||
'*'
|
||||
);
|
||||
|
||||
const { responseText, responseData } = await streamFetch({
|
||||
const { responseText } = await streamFetch({
|
||||
data: {
|
||||
messages: histories,
|
||||
variables: {
|
||||
@@ -192,7 +192,7 @@ const OutLink = (props: Props) => {
|
||||
'*'
|
||||
);
|
||||
|
||||
return { responseText, responseData, isNewChat: forbidLoadChat.current };
|
||||
return { responseText, isNewChat: forbidLoadChat.current };
|
||||
},
|
||||
[
|
||||
chatId,
|
||||
@@ -251,7 +251,7 @@ const OutLink = (props: Props) => {
|
||||
{...(isEmbed ? { p: '0 !important', borderRadius: '0', boxShadow: 'none' } : { p: [0, 5] })}
|
||||
>
|
||||
{(!quoteData || isPc) && (
|
||||
<PageContainer flex={'1 0 0'} w={0} isLoading={loading} p={'0 !important'}>
|
||||
<PageContainer flex={'1 0 0'} w={0} p={'0 !important'}>
|
||||
<Flex h={'100%'} flexDirection={['column', 'row']}>
|
||||
{RenderHistoryList}
|
||||
|
||||
@@ -391,7 +391,6 @@ export async function getServerSideProps(context: any) {
|
||||
|
||||
const app = await (async () => {
|
||||
try {
|
||||
await connectToDatabase();
|
||||
return MongoOutLink.findOne(
|
||||
{
|
||||
shareId
|
||||
|
||||
@@ -112,7 +112,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
|
||||
// Just send a user prompt
|
||||
const histories = messages.slice(-1);
|
||||
|
||||
const { responseText, responseData } = await streamFetch({
|
||||
const { responseText } = await streamFetch({
|
||||
data: {
|
||||
messages: histories,
|
||||
variables: {
|
||||
@@ -144,7 +144,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
|
||||
title: newTitle
|
||||
}));
|
||||
|
||||
return { responseText, responseData, isNewChat: forbidLoadChat.current };
|
||||
return { responseText, isNewChat: forbidLoadChat.current };
|
||||
},
|
||||
[
|
||||
chatId,
|
||||
@@ -192,13 +192,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
|
||||
)}
|
||||
|
||||
{(!quoteData || isPc) && (
|
||||
<PageContainer
|
||||
isLoading={loading}
|
||||
flex={'1 0 0'}
|
||||
w={0}
|
||||
p={[0, '16px']}
|
||||
position={'relative'}
|
||||
>
|
||||
<PageContainer flex={'1 0 0'} w={0} p={[0, '16px']} position={'relative'}>
|
||||
<Flex h={'100%'} flexDirection={['column', 'row']} bg={'white'}>
|
||||
{RenderHistoryList}
|
||||
{/* chat container */}
|
||||
|
||||
Reference in New Issue
Block a user