V4.9.4 feature (#4470)

* Training status (#4424)

* dataset data training state (#4311)

* dataset data training state

* fix

* fix ts

* fix

* fix api format

* fix

* fix

* perf: count training

* format

* fix: dataset training state (#4417)

* fix

* add test

* fix

* fix

* fix test

* fix test

* perf: training count

* count

* loading status

---------

Co-authored-by: heheer <heheer@sealos.io>

* doc

* website sync feature (#4429)

* perf: introduce BullMQ for website sync (#4403)

* perf: introduce BullMQ for website sync

* feat: new redis module

* fix: remove graceful shutdown

* perf: improve UI in dataset detail

- Updated the "change" icon SVG file.
- Modified i18n strings.
- Added new i18n string "immediate_sync".
- Improved UI in dataset detail page, including button icons and
background colors.

* refactor: Add chunkSettings to DatasetSchema

* perf: website sync ux

* env template

* fix: clean up website dataset when updating chunk settings (#4420)

* perf: check setting updated

* perf: worker currency

* feat: init script for website sync refactor (#4425)

* website feature doc

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

* pro migration (#4388) (#4433)

* pro migration

* reuse customPdfParseType

Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>

* perf: remove loading ui

* feat: config chat file expired time

* Redis cache (#4436)

* perf: add Redis cache for vector counting (#4432)

* feat: cache

* perf: get cache key

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

* perf: mobile voice input (#4437)

* update:Mobile voice interaction (#4362)

* Add files via upload

* Add files via upload

* Update ollama.md

* Update ollama.md

* Add files via upload

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update constants.ts

* Add files via upload

* Update ChatInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update useSpeech.ts

* Update ChatInput.tsx

* Add files via upload

* Update common.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update common.json

* Update common.json

* Update common.json

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update VoiceInput.tsx

* speech ui

* 优化语音输入组件,调整输入框显示逻辑,修复语音输入遮罩层样式,更新画布背景透明度,增强用户交互体验。 (#4435)

* perf: mobil voice input

---------

Co-authored-by: dreamer6680 <1468683855@qq.com>

* Test completion v2 (#4438)

* add v2 completions (#4364)

* add v2 completions

* completion config

* config version

* fix

* frontend

* doc

* fix

* fix: completions v2 api

---------

Co-authored-by: heheer <heheer@sealos.io>

* package

* Test mongo log (#4443)

* feat: mongodb-log (#4426)

* perf: mongo log

* feat: completions stop reasoner

* mongo db log

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>

* update doc

* Update doc

* fix external var ui (#4444)

* action

* fix: ts (#4458)

* preview doc action

add docs preview permission

update preview action

udpate action

* update doc (#4460)

* update preview action

* update doc

* remove

* update

* schema

* update mq export;perf: redis cache  (#4465)

* perf: redis cache

* update mq export

* perf: website sync error tip

* add error worker

* website sync ui (#4466)

* Updated the dynamic display of the voice input pop-up (#4469)

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* fix: voice input

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>
Co-authored-by: dreamer6680 <1468683855@qq.com>
Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
This commit is contained in:
Archer
2025-04-08 12:05:04 +08:00
committed by GitHub
parent 5839325f77
commit f642c9603b
151 changed files with 5434 additions and 1354 deletions

View File

@@ -1,6 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { addHours } from 'date-fns';
import { MongoImage } from '@fastgpt/service/common/file/image/schema';
@@ -56,7 +55,6 @@ async function checkInvalidImg(start: Date, end: Date, limit = 50) {
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number };

View File

@@ -1,6 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
@@ -8,7 +7,6 @@ import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
await MongoPlugin.updateMany(

View File

@@ -1,13 +1,11 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
// 删除索引

View File

@@ -1,6 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
@@ -9,7 +8,6 @@ import { POST } from '@fastgpt/service/common/api/plusRequest';
/* 初始化发布的版本 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
await MongoAppVersion.updateMany(

View File

@@ -1,6 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { addHours } from 'date-fns';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
@@ -174,7 +173,6 @@ const checkInvalidDataText = async () => {
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number };

View File

@@ -1,14 +1,11 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { MongoApp } from '@fastgpt/service/core/app/schema';
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
await MongoApp.updateMany(

View File

@@ -1,6 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant';
@@ -8,7 +7,6 @@ import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/
/* pg 中的数据搬到 mongo dataset.datas 中,并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
await MongoDataset.updateMany(

View File

@@ -0,0 +1,53 @@
import { NextAPI } from '@/service/middleware/entry';
import { retryFn } from '@fastgpt/global/common/system/utils';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { upsertWebsiteSyncJobScheduler } from '@fastgpt/service/core/dataset/websiteSync';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { addHours } from 'date-fns';
import { NextApiRequest, NextApiResponse } from 'next';
const initWebsiteSyncData = async () => {
// find out all website dataset
const datasets = await MongoDataset.find({ type: DatasetTypeEnum.websiteDataset }).lean();
console.log('更新站点同步的定时器');
// Add scheduler for all website dataset
await Promise.all(
datasets.map((dataset) => {
if (dataset.autoSync) {
// 随机生成一个往后 124 小时的时间
const time = addHours(new Date(), Math.floor(Math.random() * 23) + 1);
return retryFn(() =>
upsertWebsiteSyncJobScheduler({ datasetId: String(dataset._id) }, time.getTime())
);
}
})
);
console.log('移除站点同步集合的定时器');
// Remove all nextSyncTime
await retryFn(() =>
MongoDatasetCollection.updateMany(
{
teamId: datasets.map((dataset) => dataset.teamId),
datasetId: datasets.map((dataset) => dataset._id)
},
{
$unset: {
nextSyncTime: 1
}
}
)
);
};
async function handler(req: NextApiRequest, _res: NextApiResponse) {
await authCert({ req, authRoot: true });
await initWebsiteSyncData();
return { success: true };
}
export default NextAPI(handler);

View File

@@ -1,6 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authFileToken } from '@fastgpt/service/support/permission/controller';
import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
@@ -23,8 +22,6 @@ const previewableExtensions = [
// Abandoned, use: file/read/[filename].ts
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { token } = req.query as { token: string };
const { fileId, bucketName } = await authFileToken(token);

View File

@@ -1,6 +1,5 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authFileToken } from '@fastgpt/service/support/permission/controller';
import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
@@ -21,8 +20,6 @@ const previewableExtensions = [
];
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { token, filename } = req.query as { token: string; filename: string };
const { fileId, bucketName } = await authFileToken(token);

View File

@@ -1,5 +1,4 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { connectToDatabase } from '@/service/mongo';
import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller';
import { UploadImgProps } from '@fastgpt/global/common/file/api';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
@@ -9,7 +8,6 @@ import { NextAPI } from '@/service/middleware/entry';
Upload avatar image
*/
async function handler(req: NextApiRequest, res: NextApiResponse): Promise<string> {
await connectToDatabase();
const body = req.body as UploadImgProps;
const { teamId } = await authCert({ req, authToken: true });

View File

@@ -1,12 +1,11 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { startTrainingQueue } from '@/service/core/dataset/training/utils';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
await authCert({ req, authToken: true });
startTrainingQueue();
} catch (error) {}

View File

@@ -2,13 +2,12 @@
import { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { connectToDatabase } from '@/service/mongo';
import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
import { urlsFetch } from '@fastgpt/service/common/string/cheerio';
const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
try {
await connectToDatabase();
let { urlList = [], selector } = req.body as UrlFetchParams;
if (!urlList || urlList.length === 0) {

View File

@@ -1,6 +1,6 @@
import type { NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { pushQuestionGuideUsage } from '@/service/support/wallet/usage/push';
import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
import { ApiRequestProps } from '@fastgpt/service/type/next';
@@ -27,7 +27,6 @@ async function handler(
res: NextApiResponse<any>
) {
try {
await connectToDatabase();
const { messages } = req.body;
const { tmbId, teamId } = await authChatCert({

View File

@@ -182,7 +182,8 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
histories: newHistories,
stream: true,
maxRunTimes: WORKFLOW_MAX_RUN_TIMES,
workflowStreamResponse: workflowResponseWrite
workflowStreamResponse: workflowResponseWrite,
version: 'v2'
});
workflowResponseWrite({
@@ -197,11 +198,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
event: SseResponseEventEnum.answer,
data: '[DONE]'
});
responseWrite({
res,
event: SseResponseEventEnum.flowResponses,
data: JSON.stringify(flowResponses)
});
// save chat
const isInteractiveRequest = !!getLastInteractiveValue(histories);

View File

@@ -1,6 +1,6 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { AdminUpdateFeedbackParams } from '@/global/core/chat/api.d';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { authChatCrud } from '@/service/support/permission/auth/chat';
@@ -8,7 +8,6 @@ import { authChatCrud } from '@/service/support/permission/auth/chat';
/* 初始化我的聊天框,需要身份验证 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId, chatId, dataId, datasetId, feedbackDataId, q, a } =
req.body as AdminUpdateFeedbackParams;

View File

@@ -1,6 +1,6 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import type { CloseCustomFeedbackParams } from '@/global/core/chat/api.d';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
@@ -10,7 +10,6 @@ import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
/* remove custom feedback */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { appId, chatId, dataId, index } = req.body as CloseCustomFeedbackParams;
if (!dataId || !appId || !chatId) {

View File

@@ -1,6 +1,6 @@
import type { NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { GetChatSpeechProps } from '@/global/core/chat/api.d';
import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
import { pushAudioSpeechUsage } from '@/service/support/wallet/usage/push';
@@ -17,7 +17,6 @@ import { ApiRequestProps } from '@fastgpt/service/type/next';
*/
async function handler(req: ApiRequestProps<GetChatSpeechProps>, res: NextApiResponse) {
try {
await connectToDatabase();
const { ttsConfig, input } = req.body;
if (!ttsConfig.model || !ttsConfig.voice) {

View File

@@ -12,6 +12,9 @@ import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils';
import { getVectorCountByCollectionId } from '@fastgpt/service/common/vectorStore/controller';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { Types } from 'mongoose';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType> {
const { id } = req.query as { id: string };
@@ -30,11 +33,21 @@ async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType>
});
// get file
const [file, indexAmount] = await Promise.all([
const [file, indexAmount, errorCount] = await Promise.all([
collection?.fileId
? await getFileById({ bucketName: BucketNameEnum.dataset, fileId: collection.fileId })
: undefined,
getVectorCountByCollectionId(collection.teamId, collection.datasetId, collection._id)
getVectorCountByCollectionId(collection.teamId, collection.datasetId, collection._id),
MongoDatasetTraining.countDocuments(
{
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: id,
errorMsg: { $exists: true },
retryCount: { $lte: 0 }
},
readFromSecondary
)
]);
return {
@@ -46,7 +59,8 @@ async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType>
tags: collection.tags
}),
permission,
file
file,
errorCount
};
}

View File

@@ -93,6 +93,7 @@ async function handler(
dataAmount: 0,
indexAmount: 0,
trainingAmount: 0,
hasError: false,
permission
}))
),
@@ -113,7 +114,7 @@ async function handler(
// Compute data amount
const [trainingAmount, dataAmount]: [
{ _id: string; count: number }[],
{ _id: string; count: number; hasError: boolean }[],
{ _id: string; count: number }[]
] = await Promise.all([
MongoDatasetTraining.aggregate(
@@ -128,7 +129,8 @@ async function handler(
{
$group: {
_id: '$collectionId',
count: { $sum: 1 }
count: { $sum: 1 },
hasError: { $max: { $cond: [{ $ifNull: ['$errorMsg', false] }, true, false] } }
}
}
],
@@ -168,6 +170,7 @@ async function handler(
trainingAmount:
trainingAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
dataAmount: dataAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
hasError: trainingAmount.find((amount) => String(amount._id) === String(item._id))?.hasError,
permission
}))
);

View File

@@ -0,0 +1,170 @@
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import {
DatasetCollectionDataProcessModeEnum,
TrainingModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { NextAPI } from '@/service/middleware/entry';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { ApiRequestProps } from '@fastgpt/service/type/next';
type getTrainingDetailParams = {
collectionId: string;
};
export type getTrainingDetailResponse = {
trainingType: DatasetCollectionDataProcessModeEnum;
advancedTraining: {
customPdfParse: boolean;
imageIndex: boolean;
autoIndexes: boolean;
};
queuedCounts: Record<TrainingModeEnum, number>;
trainingCounts: Record<TrainingModeEnum, number>;
errorCounts: Record<TrainingModeEnum, number>;
trainedCount: number;
};
const defaultCounts: Record<TrainingModeEnum, number> = {
qa: 0,
chunk: 0,
image: 0,
auto: 0
};
async function handler(
req: ApiRequestProps<{}, getTrainingDetailParams>
): Promise<getTrainingDetailResponse> {
const { collectionId } = req.query;
const { collection } = await authDatasetCollection({
req,
authToken: true,
collectionId: collectionId as string,
per: ReadPermissionVal
});
const match = {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id
};
// Computed global queue
const minId = (
await MongoDatasetTraining.findOne(
{
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id
},
{ sort: { _id: 1 }, select: '_id' },
readFromSecondary
).lean()
)?._id;
const [ququedCountData, trainingCountData, errorCountData, trainedCount] = (await Promise.all([
minId
? MongoDatasetTraining.aggregate(
[
{
$match: {
_id: { $lt: minId },
retryCount: { $gt: 0 },
lockTime: { $lt: new Date('2050/1/1') }
}
},
{
$group: {
_id: '$mode',
count: { $sum: 1 }
}
}
],
readFromSecondary
)
: Promise.resolve([]),
MongoDatasetTraining.aggregate(
[
{
$match: {
...match,
retryCount: { $gt: 0 },
lockTime: { $lt: new Date('2050/1/1') }
}
},
{
$group: {
_id: '$mode',
count: { $sum: 1 }
}
}
],
readFromSecondary
),
MongoDatasetTraining.aggregate(
[
{
$match: {
...match,
retryCount: { $lte: 0 },
errorMsg: { $exists: true }
}
},
{
$group: {
_id: '$mode',
count: { $sum: 1 }
}
}
],
readFromSecondary
),
MongoDatasetData.countDocuments(match, readFromSecondary)
])) as [
{ _id: TrainingModeEnum; count: number }[],
{ _id: TrainingModeEnum; count: number }[],
{ _id: TrainingModeEnum; count: number }[],
number
];
const queuedCounts = ququedCountData.reduce(
(acc, item) => {
acc[item._id] = item.count;
return acc;
},
{ ...defaultCounts }
);
const trainingCounts = trainingCountData.reduce(
(acc, item) => {
acc[item._id] = item.count;
return acc;
},
{ ...defaultCounts }
);
const errorCounts = errorCountData.reduce(
(acc, item) => {
acc[item._id] = item.count;
return acc;
},
{ ...defaultCounts }
);
return {
trainingType: collection.trainingType,
advancedTraining: {
customPdfParse: !!collection.customPdfParse,
imageIndex: !!collection.imageIndex,
autoIndexes: !!collection.autoIndexes
},
queuedCounts,
trainingCounts,
errorCounts,
trainedCount
};
}
export default NextAPI(handler);

View File

@@ -9,6 +9,8 @@ import { OwnerPermissionVal } from '@fastgpt/global/support/permission/constant'
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema';
import { removeImageByPath } from '@fastgpt/service/common/file/image/controller';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { removeWebsiteSyncJobScheduler } from '@fastgpt/service/core/dataset/websiteSync';
async function handler(req: NextApiRequest) {
const { id: datasetId } = req.query as {
@@ -40,6 +42,13 @@ async function handler(req: NextApiRequest) {
datasetId: { $in: datasetIds }
});
await Promise.all(
datasets.map((dataset) => {
if (dataset.type === DatasetTypeEnum.websiteDataset)
return removeWebsiteSyncJobScheduler(String(dataset._id));
})
);
// delete all dataset.data and pg data
await mongoSessionRun(async (session) => {
// delete dataset data

View File

@@ -5,6 +5,8 @@ import { NextAPI } from '@/service/middleware/entry';
import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { getWebsiteSyncDatasetStatus } from '@fastgpt/service/core/dataset/websiteSync';
import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
type Query = {
id: string;
@@ -28,8 +30,21 @@ async function handler(req: ApiRequestProps<Query>): Promise<DatasetItemType> {
per: ReadPermissionVal
});
const { status, errorMsg } = await (async () => {
if (dataset.type === DatasetTypeEnum.websiteDataset) {
return await getWebsiteSyncDatasetStatus(datasetId);
}
return {
status: DatasetStatusEnum.active,
errorMsg: undefined
};
})();
return {
...dataset,
status,
errorMsg,
apiServer: dataset.apiServer
? {
baseUrl: dataset.apiServer.baseUrl,

View File

@@ -0,0 +1,39 @@
import { ManagePermissionVal } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { NextAPI } from '@/service/middleware/entry';
import { ApiRequestProps } from '@fastgpt/service/type/next';
export type deleteTrainingDataBody = {
datasetId: string;
collectionId: string;
dataId: string;
};
export type deleteTrainingDataQuery = {};
export type deleteTrainingDataResponse = {};
async function handler(
req: ApiRequestProps<deleteTrainingDataBody, deleteTrainingDataQuery>
): Promise<deleteTrainingDataResponse> {
const { datasetId, collectionId, dataId } = req.body;
const { teamId } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: ManagePermissionVal
});
await MongoDatasetTraining.deleteOne({
teamId,
datasetId,
_id: dataId
});
return {};
}
export default NextAPI(handler);

View File

@@ -0,0 +1,52 @@
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { NextAPI } from '@/service/middleware/entry';
import { ApiRequestProps } from '@fastgpt/service/type/next';
export type getTrainingDataDetailQuery = {};
export type getTrainingDataDetailBody = {
datasetId: string;
collectionId: string;
dataId: string;
};
export type getTrainingDataDetailResponse =
| {
_id: string;
datasetId: string;
mode: string;
q: string;
a: string;
}
| undefined;
async function handler(
req: ApiRequestProps<getTrainingDataDetailBody, getTrainingDataDetailQuery>
): Promise<getTrainingDataDetailResponse> {
const { datasetId, collectionId, dataId } = req.body;
const { teamId } = await authDatasetCollection({
req,
authToken: true,
collectionId,
per: ReadPermissionVal
});
const data = await MongoDatasetTraining.findOne({ teamId, datasetId, _id: dataId }).lean();
if (!data) {
return undefined;
}
return {
_id: data._id,
datasetId: data.datasetId,
mode: data.mode,
q: data.q,
a: data.a
};
}
export default NextAPI(handler);

View File

@@ -0,0 +1,51 @@
import { NextAPI } from '@/service/middleware/entry';
import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
export type getTrainingErrorBody = PaginationProps<{
collectionId: string;
}>;
export type getTrainingErrorResponse = PaginationResponse<DatasetTrainingSchemaType>;
async function handler(req: ApiRequestProps<getTrainingErrorBody, {}>) {
const { collectionId } = req.body;
const { offset, pageSize } = parsePaginationRequest(req);
const { collection } = await authDatasetCollection({
req,
authToken: true,
collectionId,
per: ReadPermissionVal
});
const match = {
teamId: collection.teamId,
datasetId: collection.datasetId,
collectionId: collection._id,
errorMsg: { $exists: true }
};
const [errorList, total] = await Promise.all([
MongoDatasetTraining.find(match, undefined, {
...readFromSecondary
})
.skip(offset)
.limit(pageSize)
.lean(),
MongoDatasetTraining.countDocuments(match, { ...readFromSecondary })
]);
return {
list: errorList,
total
};
}
export default NextAPI(handler);

View File

@@ -0,0 +1,59 @@
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
import { NextAPI } from '@/service/middleware/entry';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { addMinutes } from 'date-fns';
export type updateTrainingDataBody = {
datasetId: string;
collectionId: string;
dataId: string;
q?: string;
a?: string;
chunkIndex?: number;
};
export type updateTrainingDataQuery = {};
export type updateTrainingDataResponse = {};
async function handler(
req: ApiRequestProps<updateTrainingDataBody, updateTrainingDataQuery>
): Promise<updateTrainingDataResponse> {
const { datasetId, collectionId, dataId, q, a, chunkIndex } = req.body;
const { teamId } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: WritePermissionVal
});
const data = await MongoDatasetTraining.findOne({ teamId, datasetId, _id: dataId });
if (!data) {
return Promise.reject('data not found');
}
await MongoDatasetTraining.updateOne(
{
teamId,
datasetId,
_id: dataId
},
{
$unset: { errorMsg: '' },
retryCount: 3,
...(q !== undefined && { q }),
...(a !== undefined && { a }),
...(chunkIndex !== undefined && { chunkIndex }),
lockTime: addMinutes(new Date(), -10)
}
);
return {};
}
export default NextAPI(handler);

View File

@@ -30,6 +30,13 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection
import { addDays } from 'date-fns';
import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller';
import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import {
removeWebsiteSyncJobScheduler,
upsertWebsiteSyncJobScheduler
} from '@fastgpt/service/core/dataset/websiteSync';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller';
import { isEqual } from 'lodash';
export type DatasetUpdateQuery = {};
export type DatasetUpdateResponse = any;
@@ -62,8 +69,8 @@ async function handler(
apiServer,
yuqueServer,
feishuServer,
status,
autoSync
autoSync,
chunkSettings
} = req.body;
if (!id) {
@@ -114,6 +121,39 @@ async function handler(
});
const onUpdate = async (session: ClientSession) => {
// Website dataset update chunkSettings, need to clean up dataset
if (
dataset.type === DatasetTypeEnum.websiteDataset &&
chunkSettings &&
dataset.chunkSettings &&
!isEqual(
{
imageIndex: dataset.chunkSettings.imageIndex,
autoIndexes: dataset.chunkSettings.autoIndexes,
trainingType: dataset.chunkSettings.trainingType,
chunkSettingMode: dataset.chunkSettings.chunkSettingMode,
chunkSplitMode: dataset.chunkSettings.chunkSplitMode,
chunkSize: dataset.chunkSettings.chunkSize,
chunkSplitter: dataset.chunkSettings.chunkSplitter,
indexSize: dataset.chunkSettings.indexSize,
qaPrompt: dataset.chunkSettings.qaPrompt
},
{
imageIndex: chunkSettings.imageIndex,
autoIndexes: chunkSettings.autoIndexes,
trainingType: chunkSettings.trainingType,
chunkSettingMode: chunkSettings.chunkSettingMode,
chunkSplitMode: chunkSettings.chunkSplitMode,
chunkSize: chunkSettings.chunkSize,
chunkSplitter: chunkSettings.chunkSplitter,
indexSize: chunkSettings.indexSize,
qaPrompt: chunkSettings.qaPrompt
}
)
) {
await delDatasetRelevantData({ datasets: [dataset], session });
}
await MongoDataset.findByIdAndUpdate(
id,
{
@@ -123,7 +163,7 @@ async function handler(
...(agentModel && { agentModel }),
...(vlmModel && { vlmModel }),
...(websiteConfig && { websiteConfig }),
...(status && { status }),
...(chunkSettings && { chunkSettings }),
...(intro !== undefined && { intro }),
...(externalReadUrl !== undefined && { externalReadUrl }),
...(!!apiServer?.baseUrl && { 'apiServer.baseUrl': apiServer.baseUrl }),
@@ -143,8 +183,7 @@ async function handler(
{ session }
);
await updateSyncSchedule({
teamId: dataset.teamId,
datasetId: dataset._id,
dataset,
autoSync,
session
});
@@ -221,45 +260,54 @@ const updateTraining = async ({
};
const updateSyncSchedule = async ({
teamId,
datasetId,
dataset,
autoSync,
session
}: {
teamId: string;
datasetId: string;
dataset: DatasetSchemaType;
autoSync?: boolean;
session: ClientSession;
}) => {
if (typeof autoSync !== 'boolean') return;
// Update all collection nextSyncTime
if (autoSync) {
await MongoDatasetCollection.updateMany(
{
teamId,
datasetId,
type: { $in: [DatasetCollectionTypeEnum.apiFile, DatasetCollectionTypeEnum.link] }
},
{
$set: {
nextSyncTime: addDays(new Date(), 1)
}
},
{ session }
);
if (dataset.type === DatasetTypeEnum.websiteDataset) {
if (autoSync) {
// upsert Job Scheduler
upsertWebsiteSyncJobScheduler({ datasetId: String(dataset._id) });
} else {
// remove Job Scheduler
removeWebsiteSyncJobScheduler(String(dataset._id));
}
} else {
await MongoDatasetCollection.updateMany(
{
teamId,
datasetId
},
{
$unset: {
nextSyncTime: 1
}
},
{ session }
);
// Other dataset, update the collection sync
if (autoSync) {
await MongoDatasetCollection.updateMany(
{
teamId: dataset.teamId,
datasetId: dataset._id,
type: { $in: [DatasetCollectionTypeEnum.apiFile, DatasetCollectionTypeEnum.link] }
},
{
$set: {
nextSyncTime: addDays(new Date(), 1)
}
},
{ session }
);
} else {
await MongoDatasetCollection.updateMany(
{
teamId: dataset.teamId,
datasetId: dataset._id
},
{
$unset: {
nextSyncTime: 1
}
},
{ session }
);
}
}
};

View File

@@ -1,11 +1,10 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { request } from 'https';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { path = [], ...query } = req.query as any;
const queryStr = new URLSearchParams(query).toString();

View File

@@ -1,12 +1,11 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { request } from 'http';
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { path = [], ...query } = req.query as any;
const requestPath = `/api/${path?.join('/')}?${new URLSearchParams(query).toString()}`;

View File

@@ -2,12 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { connectToDatabase } from '@/service/mongo';
import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { oldPsw, newPsw } = req.body as { oldPsw: string; newPsw: string };
if (!oldPsw || !newPsw) {

View File

@@ -1,12 +1,11 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { size } = req.query as {
size: string;
};

View File

@@ -1,13 +1,11 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { checkWebSyncLimit } from '@fastgpt/service/support/user/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
// 凭证校验
const { teamId } = await authCert({ req, authToken: true });

View File

@@ -1,12 +1,11 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { readMongoImg } from '@fastgpt/service/common/file/image/controller';
// get the models available to the system
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { id } = req.query as { id: string };
const { binary, mime } = await readMongoImg({ id });

View File

@@ -59,7 +59,6 @@ import { getWorkflowResponseWrite } from '@fastgpt/service/core/workflow/dispatc
import { WORKFLOW_MAX_RUN_TIMES } from '@fastgpt/service/core/workflow/constants';
import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
import { ExternalProviderType } from '@fastgpt/global/core/workflow/runtime/type';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
type FastGptWebChatProps = {
chatId?: string; // undefined: get histories from messages, '': new chat, 'xxxxx': get histories from db

View File

@@ -0,0 +1,641 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { authApp } from '@fastgpt/service/support/permission/app/auth';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { sseErrRes, jsonRes } from '@fastgpt/service/common/response';
import { addLog } from '@fastgpt/service/common/system/log';
import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d';
import {
getWorkflowEntryNodeIds,
getMaxHistoryLimitFromNodes,
initWorkflowEdgeStatus,
storeNodes2RuntimeNodes,
textAdaptGptResponse,
getLastInteractiveValue
} from '@fastgpt/global/core/workflow/runtime/utils';
import { GPTMessages2Chats, chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
import { getChatItems } from '@fastgpt/service/core/chat/controller';
import { saveChat, updateInteractiveChat } from '@fastgpt/service/core/chat/saveChat';
import { responseWrite } from '@fastgpt/service/common/response';
import { createChatUsage } from '@fastgpt/service/support/wallet/usage/controller';
import { authOutLinkChatStart } from '@/service/support/permission/auth/outLink';
import { pushResult2Remote, addOutLinkUsage } from '@fastgpt/service/support/outLink/tools';
import requestIp from 'request-ip';
import { getUsageSourceByAuthType } from '@fastgpt/global/support/wallet/usage/tools';
import { authTeamSpaceToken } from '@/service/support/permission/auth/team';
import {
concatHistories,
filterPublicNodeResponseData,
getChatTitleFromChatMessage,
removeEmptyUserInput
} from '@fastgpt/global/core/chat/utils';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
import { getUserChatInfoAndAuthTeamPoints } from '@fastgpt/service/support/permission/auth/team';
import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { AppSchema } from '@fastgpt/global/core/app/type';
import { AuthOutLinkChatProps } from '@fastgpt/global/support/outLink/api';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
import { AIChatItemType, UserChatItemType } from '@fastgpt/global/core/chat/type';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
import { NextAPI } from '@/service/middleware/entry';
import { getAppLatestVersion } from '@fastgpt/service/core/app/version/controller';
import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import {
getPluginRunUserQuery,
updatePluginInputByVariables
} from '@fastgpt/global/core/workflow/utils';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { getSystemTime } from '@fastgpt/global/common/time/timezone';
import { rewriteNodeOutputByHistories } from '@fastgpt/global/core/workflow/runtime/utils';
import { getWorkflowResponseWrite } from '@fastgpt/service/core/workflow/dispatch/utils';
import { WORKFLOW_MAX_RUN_TIMES } from '@fastgpt/service/core/workflow/constants';
import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
import { ExternalProviderType } from '@fastgpt/global/core/workflow/runtime/type';
type FastGptWebChatProps = {
chatId?: string; // undefined: get histories from messages, '': new chat, 'xxxxx': get histories from db
appId?: string;
customUid?: string; // non-undefined: will be the priority provider for the logger.
metadata?: Record<string, any>;
};
export type Props = ChatCompletionCreateParams &
FastGptWebChatProps &
OutLinkChatAuthProps & {
messages: ChatCompletionMessageParam[];
responseChatItemId?: string;
stream?: boolean;
detail?: boolean;
variables: Record<string, any>; // Global variables or plugin inputs
};
type AuthResponseType = {
teamId: string;
tmbId: string;
timezone: string;
externalProvider: ExternalProviderType;
app: AppSchema;
responseDetail?: boolean;
showNodeStatus?: boolean;
authType: `${AuthUserTypeEnum}`;
apikey?: string;
responseAllData: boolean;
outLinkUserId?: string;
sourceName?: string;
};
async function handler(req: NextApiRequest, res: NextApiResponse) {
res.on('close', () => {
res.end();
});
res.on('error', () => {
console.log('error: ', 'request error');
res.end();
});
let {
chatId,
appId,
customUid,
// share chat
shareId,
outLinkUid,
// team chat
teamId: spaceTeamId,
teamToken,
stream = false,
detail = false,
messages = [],
variables = {},
responseChatItemId = getNanoid(),
metadata
} = req.body as Props;
const originIp = requestIp.getClientIp(req);
const startTime = Date.now();
try {
if (!Array.isArray(messages)) {
throw new Error('messages is not array');
}
/*
Web params: chatId + [Human]
API params: chatId + [Human]
API params: [histories, Human]
*/
const chatMessages = GPTMessages2Chats(messages);
// Computed start hook params
const startHookText = (() => {
// Chat
const userQuestion = chatMessages[chatMessages.length - 1] as UserChatItemType | undefined;
if (userQuestion) return chatValue2RuntimePrompt(userQuestion.value).text;
// plugin
return JSON.stringify(variables);
})();
/*
1. auth app permission
2. auth balance
3. get app
4. parse outLink token
*/
const {
teamId,
tmbId,
timezone,
externalProvider,
app,
responseDetail,
authType,
sourceName,
apikey,
responseAllData,
outLinkUserId = customUid,
showNodeStatus
} = await (async () => {
// share chat
if (shareId && outLinkUid) {
return authShareChat({
shareId,
outLinkUid,
chatId,
ip: originIp,
question: startHookText
});
}
// team space chat
if (spaceTeamId && appId && teamToken) {
return authTeamSpaceChat({
teamId: spaceTeamId,
teamToken,
appId,
chatId
});
}
/* parse req: api or token */
return authHeaderRequest({
req,
appId,
chatId
});
})();
const isPlugin = app.type === AppTypeEnum.plugin;
// Check message type
if (isPlugin) {
detail = true;
} else {
if (messages.length === 0) {
throw new Error('messages is empty');
}
}
// Get obj=Human history
const userQuestion: UserChatItemType = (() => {
if (isPlugin) {
return getPluginRunUserQuery({
pluginInputs: getPluginInputsFromStoreNodes(app.modules),
variables,
files: variables.files
});
}
const latestHumanChat = chatMessages.pop() as UserChatItemType | undefined;
if (!latestHumanChat) {
throw new Error('User question is empty');
}
return latestHumanChat;
})();
// Get and concat history;
const limit = getMaxHistoryLimitFromNodes(app.modules);
const [{ histories }, { nodes, edges, chatConfig }, chatDetail] = await Promise.all([
getChatItems({
appId: app._id,
chatId,
offset: 0,
limit,
field: `dataId obj value nodeOutputs`
}),
getAppLatestVersion(app._id, app),
MongoChat.findOne({ appId: app._id, chatId }, 'source variableList variables')
]);
// Get store variables(Api variable precedence)
if (chatDetail?.variables) {
variables = {
...chatDetail.variables,
...variables
};
}
// Get chat histories
const newHistories = concatHistories(histories, chatMessages);
// Get runtimeNodes
let runtimeNodes = storeNodes2RuntimeNodes(nodes, getWorkflowEntryNodeIds(nodes, newHistories));
if (isPlugin) {
// Assign values to runtimeNodes using variables
runtimeNodes = updatePluginInputByVariables(runtimeNodes, variables);
// Plugin runtime does not need global variables(It has been injected into the pluginInputNode)
variables = {};
}
runtimeNodes = rewriteNodeOutputByHistories(newHistories, runtimeNodes);
const workflowResponseWrite = getWorkflowResponseWrite({
res,
detail,
streamResponse: stream,
id: chatId,
showNodeStatus
});
/* start flow controller */
const { flowResponses, flowUsages, assistantResponses, newVariables } = await (async () => {
if (app.version === 'v2') {
return dispatchWorkFlow({
res,
requestOrigin: req.headers.origin,
mode: 'chat',
timezone,
externalProvider,
runningAppInfo: {
id: String(app._id),
teamId: String(app.teamId),
tmbId: String(app.tmbId)
},
runningUserInfo: {
teamId,
tmbId
},
uid: String(outLinkUserId || tmbId),
chatId,
responseChatItemId,
runtimeNodes,
runtimeEdges: initWorkflowEdgeStatus(edges, newHistories),
variables,
query: removeEmptyUserInput(userQuestion.value),
chatConfig,
histories: newHistories,
stream,
maxRunTimes: WORKFLOW_MAX_RUN_TIMES,
workflowStreamResponse: workflowResponseWrite,
version: 'v2'
});
}
return Promise.reject('您的工作流版本过低,请重新发布一次');
})();
// save chat
const isOwnerUse = !shareId && !spaceTeamId && String(tmbId) === String(app.tmbId);
const source = (() => {
if (shareId) {
return ChatSourceEnum.share;
}
if (authType === 'apikey') {
return ChatSourceEnum.api;
}
if (spaceTeamId) {
return ChatSourceEnum.team;
}
return ChatSourceEnum.online;
})();
const isInteractiveRequest = !!getLastInteractiveValue(histories);
const { text: userInteractiveVal } = chatValue2RuntimePrompt(userQuestion.value);
const newTitle = isPlugin
? variables.cTime ?? getSystemTime(timezone)
: getChatTitleFromChatMessage(userQuestion);
const aiResponse: AIChatItemType & { dataId?: string } = {
dataId: responseChatItemId,
obj: ChatRoleEnum.AI,
value: assistantResponses,
[DispatchNodeResponseKeyEnum.nodeResponse]: flowResponses
};
const saveChatId = chatId || getNanoid(24);
if (isInteractiveRequest) {
await updateInteractiveChat({
chatId: saveChatId,
appId: app._id,
userInteractiveVal,
aiResponse,
newVariables
});
} else {
await saveChat({
chatId: saveChatId,
appId: app._id,
teamId,
tmbId: tmbId,
nodes,
appChatConfig: chatConfig,
variables: newVariables,
isUpdateUseTime: isOwnerUse && source === ChatSourceEnum.online, // owner update use time
newTitle,
shareId,
outLinkUid: outLinkUserId,
source,
sourceName: sourceName || '',
content: [userQuestion, aiResponse],
metadata: {
originIp,
...metadata
}
});
}
addLog.info(`completions running time: ${(Date.now() - startTime) / 1000}s`);
/* select fe response field */
const feResponseData = responseAllData
? flowResponses
: filterPublicNodeResponseData({ flowResponses, responseDetail });
if (stream) {
workflowResponseWrite({
event: SseResponseEventEnum.answer,
data: textAdaptGptResponse({
text: null,
finish_reason: 'stop'
})
});
responseWrite({
res,
event: detail ? SseResponseEventEnum.answer : undefined,
data: '[DONE]'
});
res.end();
} else {
const responseContent = (() => {
if (assistantResponses.length === 0) return '';
if (assistantResponses.length === 1 && assistantResponses[0].text?.content)
return assistantResponses[0].text?.content;
if (!detail) {
return assistantResponses
.map((item) => item?.text?.content)
.filter(Boolean)
.join('\n');
}
return assistantResponses;
})();
const error = flowResponses[flowResponses.length - 1]?.error;
res.json({
...(detail ? { responseData: feResponseData, newVariables } : {}),
error,
id: chatId || '',
model: '',
usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 1 },
choices: [
{
message: { role: 'assistant', content: responseContent },
finish_reason: 'stop',
index: 0
}
]
});
}
// add record
const { totalPoints } = createChatUsage({
appName: app.name,
appId: app._id,
teamId,
tmbId: tmbId,
source: getUsageSourceByAuthType({ shareId, authType }),
flowUsages
});
if (shareId) {
pushResult2Remote({ outLinkUid, shareId, appName: app.name, flowResponses });
addOutLinkUsage({
shareId,
totalPoints
});
}
if (apikey) {
updateApiKeyUsage({
apikey,
totalPoints
});
}
} catch (err) {
if (stream) {
sseErrRes(res, err);
res.end();
} else {
jsonRes(res, {
code: 500,
error: err
});
}
}
}
export default NextAPI(handler);
const authShareChat = async ({
chatId,
...data
}: AuthOutLinkChatProps & {
shareId: string;
chatId?: string;
}): Promise<AuthResponseType> => {
const {
teamId,
tmbId,
timezone,
externalProvider,
appId,
authType,
responseDetail,
showNodeStatus,
uid,
sourceName
} = await authOutLinkChatStart(data);
const app = await MongoApp.findById(appId).lean();
if (!app) {
return Promise.reject('app is empty');
}
// get chat
const chat = await MongoChat.findOne({ appId, chatId }).lean();
if (chat && (chat.shareId !== data.shareId || chat.outLinkUid !== uid)) {
return Promise.reject(ChatErrEnum.unAuthChat);
}
return {
sourceName,
teamId,
tmbId,
app,
timezone,
externalProvider,
apikey: '',
authType,
responseAllData: false,
responseDetail,
outLinkUserId: uid,
showNodeStatus
};
};
const authTeamSpaceChat = async ({
appId,
teamId,
teamToken,
chatId
}: {
appId: string;
teamId: string;
teamToken: string;
chatId?: string;
}): Promise<AuthResponseType> => {
const { uid } = await authTeamSpaceToken({
teamId,
teamToken
});
const app = await MongoApp.findById(appId).lean();
if (!app) {
return Promise.reject('app is empty');
}
const [chat, { timezone, externalProvider }] = await Promise.all([
MongoChat.findOne({ appId, chatId }).lean(),
getUserChatInfoAndAuthTeamPoints(app.tmbId)
]);
if (chat && (String(chat.teamId) !== teamId || chat.outLinkUid !== uid)) {
return Promise.reject(ChatErrEnum.unAuthChat);
}
return {
teamId,
tmbId: app.tmbId,
app,
timezone,
externalProvider,
authType: AuthUserTypeEnum.outLink,
apikey: '',
responseAllData: false,
responseDetail: true,
outLinkUserId: uid
};
};
const authHeaderRequest = async ({
req,
appId,
chatId
}: {
req: NextApiRequest;
appId?: string;
chatId?: string;
}): Promise<AuthResponseType> => {
const {
appId: apiKeyAppId,
teamId,
tmbId,
authType,
sourceName,
apikey
} = await authCert({
req,
authToken: true,
authApiKey: true
});
const { app } = await (async () => {
if (authType === AuthUserTypeEnum.apikey) {
const currentAppId = apiKeyAppId || appId;
if (!currentAppId) {
return Promise.reject(
'Key is error. You need to use the app key rather than the account key.'
);
}
const app = await MongoApp.findById(currentAppId);
if (!app) {
return Promise.reject('app is empty');
}
appId = String(app._id);
return {
app
};
} else {
// token_auth
if (!appId) {
return Promise.reject('appId is empty');
}
const { app } = await authApp({
req,
authToken: true,
appId,
per: ReadPermissionVal
});
return {
app
};
}
})();
const [{ timezone, externalProvider }, chat] = await Promise.all([
getUserChatInfoAndAuthTeamPoints(tmbId),
MongoChat.findOne({ appId, chatId }).lean()
]);
if (
chat &&
(String(chat.teamId) !== teamId ||
// There's no need to distinguish who created it if it's apiKey auth
(authType === AuthUserTypeEnum.token && String(chat.tmbId) !== tmbId))
) {
return Promise.reject(ChatErrEnum.unAuthChat);
}
return {
teamId,
tmbId,
timezone,
externalProvider,
app,
apikey,
authType,
sourceName,
responseAllData: true,
responseDetail: true
};
};
export const config = {
api: {
bodyParser: {
sizeLimit: '20mb'
},
responseLimit: '20mb'
}
};

View File

@@ -115,7 +115,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
}: StartChatFnProps) => {
// Just send a user prompt
const histories = messages.slice(-1);
const { responseText, responseData } = await streamFetch({
const { responseText } = await streamFetch({
data: {
messages: histories,
variables,
@@ -137,7 +137,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
title: newTitle
}));
return { responseText, responseData, isNewChat: forbidLoadChat.current };
return { responseText, isNewChat: forbidLoadChat.current };
},
[appId, chatId, onUpdateHistoryTitle, setChatBoxData, forbidLoadChat]
);
@@ -174,13 +174,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
)}
{(!quoteData || isPc) && (
<PageContainer
isLoading={loading}
flex={'1 0 0'}
w={0}
p={[0, '16px']}
position={'relative'}
>
<PageContainer flex={'1 0 0'} w={0} p={[0, '16px']} position={'relative'}>
<Flex h={'100%'} flexDirection={['column', 'row']}>
{/* pc always show history. */}
{RenderHistorySlider}

View File

@@ -17,7 +17,7 @@ import { getInitOutLinkChatInfo } from '@/web/core/chat/api';
import { getChatTitleFromChatMessage } from '@fastgpt/global/core/chat/utils';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { addLog } from '@fastgpt/service/common/system/log';
import { connectToDatabase } from '@/service/mongo';
import NextHead from '@/components/common/NextHead';
import { useContextSelector } from 'use-context-selector';
import ChatContextProvider, { ChatContext } from '@/web/core/chat/context/chatContext';
@@ -151,7 +151,7 @@ const OutLink = (props: Props) => {
'*'
);
const { responseText, responseData } = await streamFetch({
const { responseText } = await streamFetch({
data: {
messages: histories,
variables: {
@@ -192,7 +192,7 @@ const OutLink = (props: Props) => {
'*'
);
return { responseText, responseData, isNewChat: forbidLoadChat.current };
return { responseText, isNewChat: forbidLoadChat.current };
},
[
chatId,
@@ -251,7 +251,7 @@ const OutLink = (props: Props) => {
{...(isEmbed ? { p: '0 !important', borderRadius: '0', boxShadow: 'none' } : { p: [0, 5] })}
>
{(!quoteData || isPc) && (
<PageContainer flex={'1 0 0'} w={0} isLoading={loading} p={'0 !important'}>
<PageContainer flex={'1 0 0'} w={0} p={'0 !important'}>
<Flex h={'100%'} flexDirection={['column', 'row']}>
{RenderHistoryList}
@@ -391,7 +391,6 @@ export async function getServerSideProps(context: any) {
const app = await (async () => {
try {
await connectToDatabase();
return MongoOutLink.findOne(
{
shareId

View File

@@ -112,7 +112,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
// Just send a user prompt
const histories = messages.slice(-1);
const { responseText, responseData } = await streamFetch({
const { responseText } = await streamFetch({
data: {
messages: histories,
variables: {
@@ -144,7 +144,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
title: newTitle
}));
return { responseText, responseData, isNewChat: forbidLoadChat.current };
return { responseText, isNewChat: forbidLoadChat.current };
},
[
chatId,
@@ -192,13 +192,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
)}
{(!quoteData || isPc) && (
<PageContainer
isLoading={loading}
flex={'1 0 0'}
w={0}
p={[0, '16px']}
position={'relative'}
>
<PageContainer flex={'1 0 0'} w={0} p={[0, '16px']} position={'relative'}>
<Flex h={'100%'} flexDirection={['column', 'row']} bg={'white'}>
{RenderHistoryList}
{/* chat container */}