Compare commits

...

8 Commits

Author SHA1 Message Date
dreamer6680
d7a722a609 add csp and more function for markdown (#4921)
* support html

* html

* add csp

* remove unuse function

---------

Co-authored-by: dreamer6680 <146868355@qq.com>
2025-05-29 17:57:37 +08:00
Archer
0f866fc552 feat: text collecion auto save for a txt file (#4924) 2025-05-29 17:57:27 +08:00
Archer
05c7ba4483 feat: Workflow node search (#4920)
* add node find (#4902)

* add node find

* plugin header

* fix

* fix

* remove

* type

* add searched status

* optimize

* perf: search nodes

---------

Co-authored-by: heheer <heheer@sealos.io>
2025-05-29 14:29:28 +08:00
heheer
fa80ce3a77 fix child app external variables (#4919) 2025-05-29 13:37:59 +08:00
Archer
830358aa72 remove invalid code (#4915) 2025-05-28 22:11:40 +08:00
Archer
02b214b3ec feat: remove buffer;fix: custom pdf parse (#4914)
* fix: doc

* fix: remove buffer

* fix: pdf parse
2025-05-28 21:48:10 +08:00
Archer
a171c7b11c perf: buffer;fix: back up split (#4913)
* perf: buffer

* fix: back up split

* fix: app limit

* doc
2025-05-28 18:18:25 +08:00
heheer
802de11363 fix runtool empty message (#4911)
* fix runtool empty message

* del unused code

* fix
2025-05-28 17:48:30 +08:00
50 changed files with 1245 additions and 217 deletions

View File

@@ -132,15 +132,15 @@ services:
# fastgpt
sandbox:
container_name: sandbox
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
networks:
- fastgpt
restart: always
fastgpt-mcp-server:
container_name: fastgpt-mcp-server
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
ports:
- 3005:3000
networks:
@@ -150,8 +150,8 @@ services:
- FASTGPT_ENDPOINT=http://fastgpt:3000
fastgpt:
container_name: fastgpt
image: ghcr.io/labring/fastgpt:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
ports:
- 3000:3000
networks:

View File

@@ -109,15 +109,15 @@ services:
# fastgpt
sandbox:
container_name: sandbox
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
networks:
- fastgpt
restart: always
fastgpt-mcp-server:
container_name: fastgpt-mcp-server
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
ports:
- 3005:3000
networks:
@@ -127,8 +127,8 @@ services:
- FASTGPT_ENDPOINT=http://fastgpt:3000
fastgpt:
container_name: fastgpt
image: ghcr.io/labring/fastgpt:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
ports:
- 3000:3000
networks:

View File

@@ -96,15 +96,15 @@ services:
# fastgpt
sandbox:
container_name: sandbox
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
networks:
- fastgpt
restart: always
fastgpt-mcp-server:
container_name: fastgpt-mcp-server
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
ports:
- 3005:3000
networks:
@@ -114,8 +114,8 @@ services:
- FASTGPT_ENDPOINT=http://fastgpt:3000
fastgpt:
container_name: fastgpt
image: ghcr.io/labring/fastgpt:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
ports:
- 3000:3000
networks:

View File

@@ -72,15 +72,15 @@ services:
sandbox:
container_name: sandbox
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
networks:
- fastgpt
restart: always
fastgpt-mcp-server:
container_name: fastgpt-mcp-server
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
ports:
- 3005:3000
networks:
@@ -90,8 +90,8 @@ services:
- FASTGPT_ENDPOINT=http://fastgpt:3000
fastgpt:
container_name: fastgpt
image: ghcr.io/labring/fastgpt:v4.9.10 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
ports:
- 3000:3000
networks:

View File

@@ -15,8 +15,8 @@ weight: 790
### 2. 更新镜像 tag
- 更新 FastGPT 镜像 tag: v4.9.10
- 更新 FastGPT 商业版镜像 tag: v4.9.10
- 更新 FastGPT 镜像 tag: v4.9.10-fix2
- 更新 FastGPT 商业版镜像 tag: v4.9.10-fix2
- mcp_server 无需更新
- Sandbox 无需更新
- AIProxy 无需更新

View File

@@ -10,12 +10,16 @@ weight: 789
## 🚀 新增内容
1. 工作流中,子流程版本控制,可选择“保持最新版本”,无需手动更新
1. 工作流中增加节点搜索功能
2. 工作流中,子流程版本控制,可选择“保持最新版本”,无需手动更新。
## ⚙️ 优化
1. 原文缓存改用 gridfs 存储,提高上限。
## 🐛 修复
1. 工作流中,管理员声明的全局系统工具,无法进行版本管理。
1. 工作流中,管理员声明的全局系统工具,无法进行版本管理。
2. 工具调用节点前,有交互节点时,上下文异常。
3. 修复备份导入,小于 1000 字时,无法分块问题。
4. 自定义 PDF 解析,无法保存 base64 图片。

View File

@@ -40,5 +40,6 @@ export function getSourceNameIcon({
export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => {
if (mode === TrainingModeEnum.qa) return data.length * 20;
if (mode === TrainingModeEnum.auto) return data.length * 5;
if (mode === TrainingModeEnum.image) return data.length * 2;
return data.length;
};

View File

@@ -125,6 +125,7 @@ export type FlowNodeItemType = FlowNodeTemplateType & {
nodeId: string;
parentNodeId?: string;
isError?: boolean;
searchedText?: string;
debugResult?: {
status: 'running' | 'success' | 'skipped' | 'failed';
message?: string;

View File

@@ -0,0 +1,178 @@
import { retryFn } from '@fastgpt/global/common/system/utils';
import { connectionMongo } from '../../mongo';
import { MongoRawTextBufferSchema, bucketName } from './schema';
import { addLog } from '../../system/log';
import { setCron } from '../../system/cron';
import { checkTimerLock } from '../../system/timerLock/utils';
import { TimerIdEnum } from '../../system/timerLock/constants';
const getGridBucket = () => {
return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db!, {
bucketName: bucketName
});
};
export const addRawTextBuffer = async ({
sourceId,
sourceName,
text,
expiredTime
}: {
sourceId: string;
sourceName: string;
text: string;
expiredTime: Date;
}) => {
const gridBucket = getGridBucket();
const metadata = {
sourceId,
sourceName,
expiredTime
};
const buffer = Buffer.from(text);
const fileSize = buffer.length;
// 单块大小:尽可能大,但不超过 14MB不小于128KB
const chunkSizeBytes = (() => {
// 计算理想块大小:文件大小 ÷ 目标块数(10)。 并且每个块需要小于 14MB
const idealChunkSize = Math.min(Math.ceil(fileSize / 10), 14 * 1024 * 1024);
// 确保块大小至少为128KB
const minChunkSize = 128 * 1024; // 128KB
// 取理想块大小和最小块大小中的较大值
let chunkSize = Math.max(idealChunkSize, minChunkSize);
// 将块大小向上取整到最接近的64KB的倍数使其更整齐
chunkSize = Math.ceil(chunkSize / (64 * 1024)) * (64 * 1024);
return chunkSize;
})();
const uploadStream = gridBucket.openUploadStream(sourceId, {
metadata,
chunkSizeBytes
});
return retryFn(async () => {
return new Promise((resolve, reject) => {
uploadStream.end(buffer);
uploadStream.on('finish', () => {
resolve(uploadStream.id);
});
uploadStream.on('error', (error) => {
addLog.error('addRawTextBuffer error', error);
resolve('');
});
});
});
};
export const getRawTextBuffer = async (sourceId: string) => {
const gridBucket = getGridBucket();
return retryFn(async () => {
const bufferData = await MongoRawTextBufferSchema.findOne(
{
'metadata.sourceId': sourceId
},
'_id metadata'
).lean();
if (!bufferData) {
return null;
}
// Read file content
const downloadStream = gridBucket.openDownloadStream(bufferData._id);
const chunks: Buffer[] = [];
return new Promise<{
text: string;
sourceName: string;
} | null>((resolve, reject) => {
downloadStream.on('data', (chunk) => {
chunks.push(chunk);
});
downloadStream.on('end', () => {
const buffer = Buffer.concat(chunks);
const text = buffer.toString('utf8');
resolve({
text,
sourceName: bufferData.metadata?.sourceName || ''
});
});
downloadStream.on('error', (error) => {
addLog.error('getRawTextBuffer error', error);
resolve(null);
});
});
});
};
export const deleteRawTextBuffer = async (sourceId: string): Promise<boolean> => {
const gridBucket = getGridBucket();
return retryFn(async () => {
const buffer = await MongoRawTextBufferSchema.findOne({ 'metadata.sourceId': sourceId });
if (!buffer) {
return false;
}
await gridBucket.delete(buffer._id);
return true;
});
};
export const updateRawTextBufferExpiredTime = async ({
sourceId,
expiredTime
}: {
sourceId: string;
expiredTime: Date;
}) => {
return retryFn(async () => {
return MongoRawTextBufferSchema.updateOne(
{ 'metadata.sourceId': sourceId },
{ $set: { 'metadata.expiredTime': expiredTime } }
);
});
};
export const clearExpiredRawTextBufferCron = async () => {
const clearExpiredRawTextBuffer = async () => {
addLog.debug('Clear expired raw text buffer start');
const gridBucket = getGridBucket();
return retryFn(async () => {
const data = await MongoRawTextBufferSchema.find(
{
'metadata.expiredTime': { $lt: new Date() }
},
'_id'
).lean();
for (const item of data) {
await gridBucket.delete(item._id);
}
addLog.debug('Clear expired raw text buffer end');
});
};
setCron('*/10 * * * *', async () => {
if (
await checkTimerLock({
timerId: TimerIdEnum.clearExpiredRawTextBuffer,
lockMinuted: 9
})
) {
try {
await clearExpiredRawTextBuffer();
} catch (error) {
addLog.error('clearExpiredRawTextBufferCron error', error);
}
}
});
};

View File

@@ -1,33 +1,22 @@
import { getMongoModel, Schema } from '../../mongo';
import { type RawTextBufferSchemaType } from './type';
import { getMongoModel, type Types, Schema } from '../../mongo';
export const collectionName = 'buffer_rawtexts';
export const bucketName = 'buffer_rawtext';
const RawTextBufferSchema = new Schema({
sourceId: {
type: String,
required: true
},
rawText: {
type: String,
default: ''
},
createTime: {
type: Date,
default: () => new Date()
},
metadata: Object
metadata: {
sourceId: { type: String, required: true },
sourceName: { type: String, required: true },
expiredTime: { type: Date, required: true }
}
});
RawTextBufferSchema.index({ 'metadata.sourceId': 'hashed' });
RawTextBufferSchema.index({ 'metadata.expiredTime': -1 });
try {
RawTextBufferSchema.index({ sourceId: 1 });
// 20 minutes
RawTextBufferSchema.index({ createTime: 1 }, { expireAfterSeconds: 20 * 60 });
} catch (error) {
console.log(error);
}
export const MongoRawTextBuffer = getMongoModel<RawTextBufferSchemaType>(
collectionName,
RawTextBufferSchema
);
export const MongoRawTextBufferSchema = getMongoModel<{
_id: Types.ObjectId;
metadata: {
sourceId: string;
sourceName: string;
expiredTime: Date;
};
}>(`${bucketName}.files`, RawTextBufferSchema);

View File

@@ -1,8 +0,0 @@
export type RawTextBufferSchemaType = {
sourceId: string;
rawText: string;
createTime: Date;
metadata?: {
filename: string;
};
};

View File

@@ -6,13 +6,13 @@ import { type DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
import { MongoChatFileSchema, MongoDatasetFileSchema } from './schema';
import { detectFileEncoding, detectFileEncodingByPath } from '@fastgpt/global/common/file/tools';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
import { readRawContentByFileBuffer } from '../read/utils';
import { gridFsStream2Buffer, stream2Encoding } from './utils';
import { addLog } from '../../system/log';
import { readFromSecondary } from '../../mongo/utils';
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
import { Readable } from 'stream';
import { addRawTextBuffer, getRawTextBuffer } from '../../buffer/rawText/controller';
import { addMinutes } from 'date-fns';
export function getGFSCollection(bucket: `${BucketNameEnum}`) {
MongoDatasetFileSchema;
@@ -223,15 +223,13 @@ export const readFileContentFromMongo = async ({
rawText: string;
filename: string;
}> => {
const bufferId = `${fileId}-${customPdfParse}`;
const bufferId = `${String(fileId)}-${customPdfParse}`;
// read buffer
const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: bufferId }, undefined, {
...readFromSecondary
}).lean();
const fileBuffer = await getRawTextBuffer(bufferId);
if (fileBuffer) {
return {
rawText: fileBuffer.rawText,
filename: fileBuffer.metadata?.filename || ''
rawText: fileBuffer.text,
filename: fileBuffer?.sourceName
};
}
@@ -265,16 +263,13 @@ export const readFileContentFromMongo = async ({
}
});
// < 14M
if (fileBuffers.length < 14 * 1024 * 1024 && rawText.trim()) {
MongoRawTextBuffer.create({
sourceId: bufferId,
rawText,
metadata: {
filename: file.filename
}
});
}
// Add buffer
addRawTextBuffer({
sourceId: bufferId,
sourceName: file.filename,
text: rawText,
expiredTime: addMinutes(new Date(), 20)
});
return {
rawText,

View File

@@ -1,16 +1,16 @@
import { Schema, getMongoModel } from '../../mongo';
const DatasetFileSchema = new Schema({});
const ChatFileSchema = new Schema({});
const DatasetFileSchema = new Schema({
metadata: Object
});
const ChatFileSchema = new Schema({
metadata: Object
});
try {
DatasetFileSchema.index({ uploadDate: -1 });
DatasetFileSchema.index({ uploadDate: -1 });
ChatFileSchema.index({ uploadDate: -1 });
ChatFileSchema.index({ 'metadata.chatId': 1 });
} catch (error) {
console.log(error);
}
ChatFileSchema.index({ uploadDate: -1 });
ChatFileSchema.index({ 'metadata.chatId': 1 });
export const MongoDatasetFileSchema = getMongoModel('dataset.files', DatasetFileSchema);
export const MongoChatFileSchema = getMongoModel('chat.files', ChatFileSchema);

View File

@@ -1,5 +1,57 @@
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
import { PassThrough } from 'stream';
import { getGridBucket } from './controller';
import { type BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { retryFn } from '@fastgpt/global/common/system/utils';
export const createFileFromText = async ({
bucket,
filename,
text,
metadata
}: {
bucket: `${BucketNameEnum}`;
filename: string;
text: string;
metadata: Record<string, any>;
}) => {
const gridBucket = getGridBucket(bucket);
const buffer = Buffer.from(text);
const fileSize = buffer.length;
// 单块大小:尽可能大,但不超过 14MB不小于128KB
const chunkSizeBytes = (() => {
// 计算理想块大小:文件大小 ÷ 目标块数(10)。 并且每个块需要小于 14MB
const idealChunkSize = Math.min(Math.ceil(fileSize / 10), 14 * 1024 * 1024);
// 确保块大小至少为128KB
const minChunkSize = 128 * 1024; // 128KB
// 取理想块大小和最小块大小中的较大值
let chunkSize = Math.max(idealChunkSize, minChunkSize);
// 将块大小向上取整到最接近的64KB的倍数使其更整齐
chunkSize = Math.ceil(chunkSize / (64 * 1024)) * (64 * 1024);
return chunkSize;
})();
const uploadStream = gridBucket.openUploadStream(filename, {
metadata,
chunkSizeBytes
});
return retryFn(async () => {
return new Promise<{ fileId: string }>((resolve, reject) => {
uploadStream.end(buffer);
uploadStream.on('finish', () => {
resolve({ fileId: String(uploadStream.id) });
});
uploadStream.on('error', reject);
});
});
};
export const gridFsStream2Buffer = (stream: NodeJS.ReadableStream) => {
return new Promise<Buffer>((resolve, reject) => {

View File

@@ -110,7 +110,7 @@ export const readRawContentByFileBuffer = async ({
return {
rawText: text,
formatText: rawText,
formatText: text,
imageList
};
};

View File

@@ -5,7 +5,8 @@ export enum TimerIdEnum {
clearExpiredSubPlan = 'clearExpiredSubPlan',
updateStandardPlan = 'updateStandardPlan',
scheduleTriggerApp = 'scheduleTriggerApp',
notification = 'notification'
notification = 'notification',
clearExpiredRawTextBuffer = 'clearExpiredRawTextBuffer'
}
export enum LockNotificationEnum {

View File

@@ -77,7 +77,10 @@ export const createCollectionAndInsertData = async ({
const chunkSplitter = computeChunkSplitter(createCollectionParams);
const paragraphChunkDeep = computeParagraphChunkDeep(createCollectionParams);
if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
if (
trainingType === DatasetCollectionDataProcessModeEnum.qa ||
trainingType === DatasetCollectionDataProcessModeEnum.backup
) {
delete createCollectionParams.chunkTriggerType;
delete createCollectionParams.chunkTriggerMinSize;
delete createCollectionParams.dataEnhanceCollectionName;

View File

@@ -218,6 +218,10 @@ export const rawText2Chunks = ({
};
};
if (backupParse) {
return parseDatasetBackup2Chunks(rawText).chunks;
}
// Chunk condition
// 1. 选择最大值条件,只有超过了最大值(默认为模型的最大值*0.7),才会触发分块
if (chunkTriggerType === ChunkTriggerConfigTypeEnum.maxSize) {
@@ -240,10 +244,6 @@ export const rawText2Chunks = ({
}
}
if (backupParse) {
return parseDatasetBackup2Chunks(rawText).chunks;
}
const { chunks } = splitText2Chunks({
text: rawText,
chunkSize,

View File

@@ -86,7 +86,6 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
});
// Check interactive entry
const interactiveResponse = lastInteractive;
props.node.isEntry = false;
const hasReadFilesTool = toolNodes.some(
(item) => item.flowNodeType === FlowNodeTypeEnum.readFiles
@@ -143,7 +142,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
})
}
];
if (interactiveResponse) {
if (lastInteractive && isEntry) {
return value.slice(0, -2);
}
return value;
@@ -183,7 +182,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
toolModel,
maxRunToolTimes: 30,
messages: adaptMessages,
interactiveEntryToolParams: interactiveResponse?.toolParams
interactiveEntryToolParams: lastInteractive?.toolParams
});
}
if (toolModel.functionCall) {
@@ -194,7 +193,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
toolNodes,
toolModel,
messages: adaptMessages,
interactiveEntryToolParams: interactiveResponse?.toolParams
interactiveEntryToolParams: lastInteractive?.toolParams
});
}
@@ -224,7 +223,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
toolNodes,
toolModel,
messages: adaptMessages,
interactiveEntryToolParams: interactiveResponse?.toolParams
interactiveEntryToolParams: lastInteractive?.toolParams
});
})();

View File

@@ -11,7 +11,6 @@ import type {
SystemVariablesType
} from '@fastgpt/global/core/workflow/runtime/type';
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type.d';
import type { FlowNodeOutputItemType } from '@fastgpt/global/core/workflow/type/io.d';
import type {
AIChatItemValueItemType,
ChatHistoryItemResType,

View File

@@ -17,6 +17,7 @@ import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
import { getPluginRunUserQuery } from '@fastgpt/global/core/workflow/utils';
import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { getUserChatInfoAndAuthTeamPoints } from '../../../../support/permission/auth/team';
type RunPluginProps = ModuleDispatchProps<{
[NodeInputKeyEnum.forbidStream]?: boolean;
@@ -73,9 +74,11 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
};
});
const { externalProvider } = await getUserChatInfoAndAuthTeamPoints(runningAppInfo.tmbId);
const runtimeVariables = {
...filterSystemVariables(props.variables),
appId: String(plugin.id)
appId: String(plugin.id),
...(externalProvider ? externalProvider.externalWorkflowVariables : {})
};
const { flowResponses, flowUsages, assistantResponses, runTimes } = await dispatchWorkFlow({
...props,

View File

@@ -20,6 +20,7 @@ import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { getAppVersionById } from '../../../app/version/controller';
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { type ChildrenInteractive } from '@fastgpt/global/core/workflow/template/system/interactive/type';
import { getUserChatInfoAndAuthTeamPoints } from '../../../../support/permission/auth/team';
type Props = ModuleDispatchProps<{
[NodeInputKeyEnum.userChatInput]: string;
@@ -97,11 +98,13 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
// Rewrite children app variables
const systemVariables = filterSystemVariables(variables);
const { externalProvider } = await getUserChatInfoAndAuthTeamPoints(appData.tmbId);
const childrenRunVariables = {
...systemVariables,
...childrenAppVariables,
histories: chatHistories,
appId: String(appData._id)
appId: String(appData._id),
...(externalProvider ? externalProvider.externalWorkflowVariables : {})
};
const childrenInteractive =

View File

@@ -5,8 +5,6 @@ import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
import axios from 'axios';
import { serverRequestBaseUrl } from '../../../../common/api/serverRequest';
import { MongoRawTextBuffer } from '../../../../common/buffer/rawText/schema';
import { readFromSecondary } from '../../../../common/mongo/utils';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { detectFileEncoding, parseUrlToFileType } from '@fastgpt/global/common/file/tools';
import { readRawContentByFileBuffer } from '../../../../common/file/read/utils';
@@ -14,6 +12,8 @@ import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
import { type ChatItemType, type UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
import { addLog } from '../../../../common/system/log';
import { addRawTextBuffer, getRawTextBuffer } from '../../../../common/buffer/rawText/controller';
import { addMinutes } from 'date-fns';
type Props = ModuleDispatchProps<{
[NodeInputKeyEnum.fileUrlList]: string[];
@@ -158,14 +158,12 @@ export const getFileContentFromLinks = async ({
parseUrlList
.map(async (url) => {
// Get from buffer
const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: url }, undefined, {
...readFromSecondary
}).lean();
const fileBuffer = await getRawTextBuffer(url);
if (fileBuffer) {
return formatResponseObject({
filename: fileBuffer.metadata?.filename || url,
filename: fileBuffer.sourceName || url,
url,
content: fileBuffer.rawText
content: fileBuffer.text
});
}
@@ -220,17 +218,12 @@ export const getFileContentFromLinks = async ({
});
// Add to buffer
try {
if (buffer.length < 14 * 1024 * 1024 && rawText.trim()) {
MongoRawTextBuffer.create({
sourceId: url,
rawText,
metadata: {
filename: filename
}
});
}
} catch (error) {}
addRawTextBuffer({
sourceId: url,
sourceName: filename,
text: rawText,
expiredTime: addMinutes(new Date(), 20)
});
return formatResponseObject({ filename, url, content: rawText });
} catch (error) {

View File

@@ -1,17 +1,26 @@
import { Box } from '@chakra-ui/react';
import React from 'react';
import React, { useMemo } from 'react';
const HighlightText = ({
rawText,
matchText,
color = 'primary.600'
color = 'primary.600',
mode = 'text'
}: {
rawText: string;
matchText: string;
color?: string;
mode?: 'text' | 'bg';
}) => {
const regex = new RegExp(`(${matchText})`, 'gi');
const parts = rawText.split(regex);
const { parts } = useMemo(() => {
const regx = new RegExp(`(${matchText})`, 'gi');
const parts = rawText.split(regx);
return {
regx,
parts
};
}, [rawText, matchText]);
return (
<Box>
@@ -28,7 +37,17 @@ const HighlightText = ({
}
return (
<Box as="span" key={index} color={highLight ? color : 'inherit'}>
<Box
as="span"
key={index}
{...(mode === 'bg'
? {
bg: highLight ? color : 'transparent'
}
: {
color: highLight ? color : 'inherit'
})}
>
{part}
</Box>
);
@@ -37,4 +56,4 @@ const HighlightText = ({
);
};
export default HighlightText;
export default React.memo(HighlightText);

View File

@@ -3,6 +3,8 @@ import { useContextSelector } from 'use-context-selector';
export const useSystem = () => {
const isPc = useContextSelector(useSystemStoreContext, (state) => state.isPc);
const isMac =
typeof window !== 'undefined' && window.navigator.userAgent.toLocaleLowerCase().includes('mac');
return { isPc };
return { isPc, isMac };
};

View File

@@ -63,6 +63,8 @@
"field_required": "Required",
"field_used_as_tool_input": "Used as Tool Call Parameter",
"filter_description": "Currently supports filtering by tags and creation time. Fill in the format as follows:\n{\n \"tags\": {\n \"$and\": [\"Tag 1\",\"Tag 2\"],\n \"$or\": [\"When there are $and tags, and is effective, or is not effective\"]\n },\n \"createTime\": {\n \"$gte\": \"YYYY-MM-DD HH:mm format, collection creation time greater than this time\",\n \"$lte\": \"YYYY-MM-DD HH:mm format, collection creation time less than this time, can be used with $gte\"\n }\n}",
"find_tip": "Find node ctrl f",
"find_tip_mac": "Find node ⌘ f",
"foldAll": "Collapse all",
"form_input_result": "User complete input result",
"form_input_result_tip": "an object containing the complete result",
@@ -123,18 +125,23 @@
"max_tokens": "Maximum Tokens",
"mouse_priority": "Mouse first\n- Press the left button to drag the canvas\n- Hold down shift and left click to select batches",
"new_context": "New Context",
"next": "Next",
"no_match_node": "No results",
"no_node_found": "No node was not found",
"not_contains": "Does Not Contain",
"only_the_reference_type_is_supported": "Only reference type is supported",
"optional_value_type": "Optional Value Type",
"optional_value_type_tip": "You can specify one or more data types. When dynamically adding fields, users can only select the configured types.",
"pan_priority": "Touchpad first\n- Click to batch select\n- Move the canvas with two fingers",
"pass_returned_object_as_output_to_next_nodes": "Pass the object returned in the code as output to the next nodes. The variable name needs to correspond to the return key.",
"please_enter_node_name": "Enter the node name",
"plugin.Instruction_Tip": "You can configure an instruction to explain the purpose of the plugin. This instruction will be displayed each time the plugin is used. Supports standard Markdown syntax.",
"plugin.Instructions": "Instructions",
"plugin.global_file_input": "File links (deprecated)",
"plugin_file_abandon_tip": "Plugin global file upload has been deprecated, please adjust it as soon as possible. \nRelated functions can be achieved through plug-in input and adding image type input.",
"plugin_input": "Plugin Input",
"plugin_output_tool": "When the plug-in is executed as a tool, whether this field responds as a result of the tool",
"previous": "Previous",
"question_classification": "Classify",
"question_optimization": "Query extension",
"quote_content_placeholder": "The structure of the reference content can be customized to better suit different scenarios. \nSome variables can be used for template configuration\n\n{{q}} - main content\n\n{{a}} - auxiliary data\n\n{{source}} - source name\n\n{{sourceId}} - source ID\n\n{{index}} - nth reference",
@@ -177,9 +184,9 @@
"text_content_extraction": "Text Extract",
"text_to_extract": "Text to Extract",
"these_variables_will_be_input_parameters_for_code_execution": "These variables will be input parameters for code execution",
"tool.tool_result": "Tool operation results",
"to_add_node": "to add",
"to_connect_node": "to connect",
"tool.tool_result": "Tool operation results",
"tool_call_termination": "Stop ToolCall",
"tool_custom_field": "Custom Tool",
"tool_field": " Tool Field Parameter Configuration",

View File

@@ -63,6 +63,8 @@
"field_required": "必填",
"field_used_as_tool_input": "作为工具调用参数",
"filter_description": "目前支持标签和创建时间过滤,需按照以下格式填写:\n{\n \"tags\": {\n \"$and\": [\"标签 1\",\"标签 2\"],\n \"$or\": [\"有 $and 标签时and 生效or 不生效\"]\n },\n \"createTime\": {\n \"$gte\": \"YYYY-MM-DD HH:mm 格式即可,集合的创建时间大于该时间\",\n \"$lte\": \"YYYY-MM-DD HH:mm 格式即可,集合的创建时间小于该时间,可和 $gte 共同使用\"\n }\n}",
"find_tip": "查找节点 ctrl f",
"find_tip_mac": "查找节点 ⌘ f",
"foldAll": "全部折叠",
"form_input_result": "用户完整输入结果",
"form_input_result_tip": "一个包含完整结果的对象",
@@ -123,18 +125,23 @@
"max_tokens": "最大 Tokens",
"mouse_priority": "鼠标优先\n- 左键按下后可拖动画布\n- 按住 shift 后左键可批量选择",
"new_context": "新的上下文",
"next": "下一个",
"no_match_node": "无结果",
"no_node_found": "未搜索到节点",
"not_contains": "不包含",
"only_the_reference_type_is_supported": "仅支持引用类型",
"optional_value_type": "可选的数据类型",
"optional_value_type_tip": "可以指定 1 个或多个数据类型,用户在动态添加字段时,仅可选择配置的类型",
"pan_priority": "触摸板优先\n- 单击批量选择\n- 双指移动画布",
"pass_returned_object_as_output_to_next_nodes": "将代码中 return 的对象作为输出,传递给后续的节点。变量名需要对应 return 的 key",
"please_enter_node_name": "请输入节点名称",
"plugin.Instruction_Tip": "可以配置一段说明,以解释该插件的用途。每次使用插件前,会显示该段说明。支持标准 Markdown 语法。",
"plugin.Instructions": "使用说明",
"plugin.global_file_input": "文件链接(弃用)",
"plugin_file_abandon_tip": "插件全局文件上传已弃用,请尽快调整。可以通过插件输入,添加图片类型输入来实现相关功能。",
"plugin_input": "插件输入",
"plugin_output_tool": "插件作为工具执行时,该字段是否作为工具响应结果",
"previous": "上一个",
"question_classification": "问题分类",
"question_optimization": "问题优化",
"quote_content_placeholder": "可以自定义引用内容的结构,以更好的适配不同场景。可以使用一些变量来进行模板配置\n{{q}} - 主要内容\n{{a}} - 辅助数据\n{{source}} - 来源名\n{{sourceId}} - 来源ID\n{{index}} - 第 n 个引用",

View File

@@ -63,6 +63,8 @@
"field_required": "必填",
"field_used_as_tool_input": "作為工具呼叫參數",
"filter_description": "目前支援標籤和建立時間篩選,需按照以下格式填寫:\n{\n \"tags\": {\n \"$and\": [\"標籤 1\",\"標籤 2\"],\n \"$or\": [\"當有 $and 標籤時,$and 才會生效,$or 不會生效\"]\n },\n \"createTime\": {\n \"$gte\": \"YYYY-MM-DD HH:mm 格式,資料集的建立時間大於這個時間\",\n \"$lte\": \"YYYY-MM-DD HH:mm 格式,資料集的建立時間小於這個時間,可以和 $gte 一起使用\"\n }\n}",
"find_tip": "查找節點 ctrl f",
"find_tip_mac": "查找節點 ⌘ f",
"foldAll": "全部折疊",
"form_input_result": "使用者完整輸入結果",
"form_input_result_tip": "一個包含完整結果的物件",
@@ -123,18 +125,23 @@
"max_tokens": "最大 Token 數",
"mouse_priority": "滑鼠優先\n- 按下左鍵拖曳畫布\n- 按住 Shift 鍵並點選左鍵可批次選取",
"new_context": "新的脈絡",
"next": "下一個",
"no_match_node": "無結果",
"no_node_found": "未搜索到節點",
"not_contains": "不包含",
"only_the_reference_type_is_supported": "僅支援引用類型",
"optional_value_type": "可選的資料類型",
"optional_value_type_tip": "可以指定一或多個資料類型,使用者在動態新增欄位時,只能選擇已設定的類型",
"pan_priority": "觸控板優先\n- 點選可批次選取\n- 使用兩指移動畫布",
"pass_returned_object_as_output_to_next_nodes": "將程式碼中 return 的物件作為輸出,傳遞給後續的節點。變數名稱需要對應 return 的鍵值",
"please_enter_node_name": "請輸入節點名稱",
"plugin.Instruction_Tip": "您可以設定一段說明來解釋這個外掛程式的用途。每次使用外掛程式前,都會顯示這段說明。支援標準 Markdown 語法。",
"plugin.Instructions": "使用說明",
"plugin.global_file_input": "檔案連結(已淘汰)",
"plugin_file_abandon_tip": "外掛程式全域檔案上傳功能已淘汰,請儘速調整。您可以透過外掛程式輸入,新增圖片類型輸入來達成相關功能。",
"plugin_input": "外掛程式輸入",
"plugin_output_tool": "外掛程式作為工具執行時,這個欄位是否作為工具的回應結果",
"previous": "上一個",
"question_classification": "問題分類",
"question_optimization": "問題最佳化",
"quote_content_placeholder": "可以自訂引用內容的結構,以便更好地適應不同場景。可以使用一些變數來設定範本\n{{q}} - 主要內容\n{{a}} - 輔助資料\n{{source}} - 來源名稱\n{{sourceId}} - 來源 ID\n{{index}} - 第 n 個引用",
@@ -177,9 +184,9 @@
"text_content_extraction": "文字內容擷取",
"text_to_extract": "要擷取的文字",
"these_variables_will_be_input_parameters_for_code_execution": "這些變數會作為程式碼執行的輸入參數",
"tool.tool_result": "工具運行結果",
"to_add_node": "添加節點",
"to_connect_node": "連接節點",
"tool.tool_result": "工具運行結果",
"tool_call_termination": "工具呼叫終止",
"tool_custom_field": "自訂工具變數",
"tool_field": "工具參數設定",

68
pnpm-lock.yaml generated
View File

@@ -46,7 +46,7 @@ importers:
version: 10.1.4(socks@2.8.4)
next-i18next:
specifier: 15.4.2
version: 15.4.2(i18next@23.16.8)(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
version: 15.4.2(i18next@23.16.8)(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
prettier:
specifier: 3.2.4
version: 3.2.4
@@ -343,7 +343,7 @@ importers:
version: 2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)
'@chakra-ui/next-js':
specifier: 2.4.2
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
'@chakra-ui/react':
specifier: 2.10.7
version: 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -406,7 +406,7 @@ importers:
version: 4.17.21
next-i18next:
specifier: 15.4.2
version: 15.4.2(i18next@23.16.8)(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
version: 15.4.2(i18next@23.16.8)(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
papaparse:
specifier: ^5.4.1
version: 5.4.1
@@ -467,7 +467,7 @@ importers:
version: 2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)
'@chakra-ui/next-js':
specifier: 2.4.2
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
'@chakra-ui/react':
specifier: 2.10.7
version: 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@@ -569,7 +569,7 @@ importers:
version: 14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
next-i18next:
specifier: 15.4.2
version: 15.4.2(i18next@23.16.8)(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
version: 15.4.2(i18next@23.16.8)(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
nprogress:
specifier: ^0.2.0
version: 0.2.0
@@ -612,6 +612,9 @@ importers:
rehype-katex:
specifier: ^7.0.0
version: 7.0.1
rehype-raw:
specifier: ^7.0.0
version: 7.0.0
remark-breaks:
specifier: ^4.0.0
version: 4.0.0
@@ -5879,9 +5882,15 @@ packages:
hast-util-parse-selector@4.0.0:
resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==}
hast-util-raw@9.1.0:
resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==}
hast-util-to-jsx-runtime@2.3.6:
resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==}
hast-util-to-parse5@8.0.0:
resolution: {integrity: sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw==}
hast-util-to-text@4.0.2:
resolution: {integrity: sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==}
@@ -5919,6 +5928,9 @@ packages:
html-url-attributes@3.0.1:
resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==}
html-void-elements@3.0.0:
resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==}
htmlparser2@8.0.2:
resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==}
@@ -8026,6 +8038,9 @@ packages:
property-information@5.6.0:
resolution: {integrity: sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==}
property-information@6.5.0:
resolution: {integrity: sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==}
property-information@7.0.0:
resolution: {integrity: sha512-7D/qOz/+Y4X/rzSB6jKxKUsQnphO046ei8qxG59mtM3RG3DHgTK81HrxrmoDVINJb8NKT5ZsRbwHvQ6B68Iyhg==}
@@ -8361,6 +8376,9 @@ packages:
rehype-katex@7.0.1:
resolution: {integrity: sha512-OiM2wrZ/wuhKkigASodFoo8wimG3H12LWQaH8qSPVJn9apWKFSH3YOCtbKpBorTVw/eI7cuT21XBbvwEswbIOA==}
rehype-raw@7.0.0:
resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==}
remark-breaks@4.0.0:
resolution: {integrity: sha512-IjEjJOkH4FuJvHZVIW0QCDWxcG96kCq7An/KVH2NfJe6rKZU2AsHeB3OEjPNRxi4QC34Xdx7I2KGYn6IpT7gxQ==}
@@ -10969,7 +10987,7 @@ snapshots:
'@chakra-ui/system': 2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1)
react: 18.3.1
'@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
'@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
dependencies:
'@chakra-ui/react': 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
'@emotion/cache': 11.14.0
@@ -16108,6 +16126,22 @@ snapshots:
dependencies:
'@types/hast': 3.0.4
hast-util-raw@9.1.0:
dependencies:
'@types/hast': 3.0.4
'@types/unist': 3.0.3
'@ungap/structured-clone': 1.3.0
hast-util-from-parse5: 8.0.3
hast-util-to-parse5: 8.0.0
html-void-elements: 3.0.0
mdast-util-to-hast: 13.2.0
parse5: 7.2.1
unist-util-position: 5.0.0
unist-util-visit: 5.0.0
vfile: 6.0.3
web-namespaces: 2.0.1
zwitch: 2.0.4
hast-util-to-jsx-runtime@2.3.6:
dependencies:
'@types/estree': 1.0.6
@@ -16128,6 +16162,16 @@ snapshots:
transitivePeerDependencies:
- supports-color
hast-util-to-parse5@8.0.0:
dependencies:
'@types/hast': 3.0.4
comma-separated-tokens: 2.0.3
devlop: 1.1.0
property-information: 6.5.0
space-separated-tokens: 2.0.2
web-namespaces: 2.0.1
zwitch: 2.0.4
hast-util-to-text@4.0.2:
dependencies:
'@types/hast': 3.0.4
@@ -16175,6 +16219,8 @@ snapshots:
html-url-attributes@3.0.1: {}
html-void-elements@3.0.0: {}
htmlparser2@8.0.2:
dependencies:
domelementtype: 2.3.0
@@ -18238,7 +18284,7 @@ snapshots:
transitivePeerDependencies:
- supports-color
next-i18next@15.4.2(i18next@23.16.8)(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
next-i18next@15.4.2(i18next@23.16.8)(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
dependencies:
'@babel/runtime': 7.26.10
'@types/hoist-non-react-statics': 3.3.6
@@ -18873,6 +18919,8 @@ snapshots:
dependencies:
xtend: 4.0.2
property-information@6.5.0: {}
property-information@7.0.0: {}
proto-list@1.2.4: {}
@@ -19292,6 +19340,12 @@ snapshots:
unist-util-visit-parents: 6.0.1
vfile: 6.0.3
rehype-raw@7.0.0:
dependencies:
'@types/hast': 3.0.4
hast-util-raw: 9.1.0
vfile: 6.0.3
remark-breaks@4.0.0:
dependencies:
'@types/mdast': 4.0.4

View File

@@ -11,6 +11,53 @@ const nextConfig = {
output: 'standalone',
reactStrictMode: isDev ? false : true,
compress: true,
headers: async () => {
const nonce = Buffer.from(crypto.randomUUID()).toString('base64');
const csp = `'nonce-${nonce}'`;
const scheme_source = 'data: mediastream: blob: filesystem:';
const NECESSARY_DOMAINS = [
'*.sentry.io',
'http://localhost:*',
'http://127.0.0.1:*',
'https://analytics.google.com',
'googletagmanager.com',
'*.googletagmanager.com',
'https://www.google-analytics.com',
'https://api.github.com'
].join(' ');
return [
{
source: '/chat/(.*)',
headers: [
{ key: 'X-Frame-Options', value: 'DENY' },
{ key: 'X-Content-Type-Options', value: 'nosniff' },
{ key: 'X-XSS-Protection', value: '1; mode=block' },
{ key: 'Referrer-Policy', value: 'no-referrer' },
{
key: 'Content-Security-Policy',
value: [
`default-src 'self' ${scheme_source} ${NECESSARY_DOMAINS} ${csp}`,
`script-src 'self' 'unsafe-inline' 'unsafe-eval' ${csp} ${NECESSARY_DOMAINS}`,
`style-src 'self' 'unsafe-inline' ${csp} ${NECESSARY_DOMAINS}`,
`media-src 'self' http: ${scheme_source} ${NECESSARY_DOMAINS} ${csp}`,
`worker-src 'self' ${csp} ${NECESSARY_DOMAINS} ${scheme_source}`,
`img-src * data: blob:`,
`font-src 'self'`,
`connect-src 'self' wss: https: ${scheme_source} ${NECESSARY_DOMAINS} ${csp}`,
"object-src 'none'",
"form-action 'self'",
"base-uri 'self'",
"frame-src 'self' 'allow-scripts'",
'sandbox allow-scripts allow-same-origin allow-popups allow-forms',
'upgrade-insecure-requests'
].join('; ')
}
]
}
];
},
webpack(config, { isServer, nextRuntime }) {
Object.assign(config.resolve.alias, {
'@mongodb-js/zstd': false,
@@ -85,7 +132,7 @@ const nextConfig = {
'pg',
'bullmq',
'@zilliz/milvus2-sdk-node',
"tiktoken",
'tiktoken'
],
outputFileTracingRoot: path.join(__dirname, '../../'),
instrumentationHook: true

View File

@@ -60,6 +60,7 @@
"recharts": "^2.15.0",
"rehype-external-links": "^3.0.0",
"rehype-katex": "^7.0.0",
"rehype-raw": "^7.0.0",
"remark-breaks": "^4.0.0",
"remark-gfm": "^4.0.0",
"remark-math": "^6.0.0",

View File

@@ -0,0 +1,35 @@
import React from 'react';
interface ErrorBoundaryProps {
children: React.ReactNode;
fallback?: React.ReactNode;
}
interface ErrorBoundaryState {
hasError: boolean;
error: Error | null;
}
class ErrorBoundary extends React.Component<ErrorBoundaryProps, ErrorBoundaryState> {
constructor(props: ErrorBoundaryProps) {
super(props);
this.state = { hasError: false, error: null };
}
static getDerivedStateFromError(error: Error) {
return { hasError: true, error };
}
componentDidCatch(error: Error, errorInfo: React.ErrorInfo) {
console.error('ErrorBoundary caught an error:', error, errorInfo);
}
render() {
if (this.state.hasError) {
return this.props.fallback || <div>Something went wrong while rendering Markdown.</div>;
}
return this.props.children;
}
}
export default ErrorBoundary;

View File

@@ -1,4 +1,4 @@
import React, { useCallback, useMemo } from 'react';
import React, { memo, useCallback, useMemo } from 'react';
import ReactMarkdown from 'react-markdown';
import 'katex/dist/katex.min.css';
import RemarkMath from 'remark-math'; // Math syntax
@@ -6,15 +6,15 @@ import RemarkBreaks from 'remark-breaks'; // Line break
import RehypeKatex from 'rehype-katex'; // Math render
import RemarkGfm from 'remark-gfm'; // Special markdown syntax
import RehypeExternalLinks from 'rehype-external-links';
import RehypeRaw from 'rehype-raw';
import styles from './index.module.scss';
import dynamic from 'next/dynamic';
import { Box } from '@chakra-ui/react';
import { CodeClassNameEnum, mdTextFormat } from './utils';
import ErrorBoundary from './errorBoundry';
import SVGRenderer from './markdowSVG';
import { useCreation } from 'ahooks';
import type { AProps } from './A';
const CodeLight = dynamic(() => import('./codeBlock/CodeLight'), { ssr: false });
const MermaidCodeBlock = dynamic(() => import('./img/MermaidCodeBlock'), { ssr: false });
const MdImage = dynamic(() => import('./img/Image'), { ssr: false });
@@ -26,7 +26,40 @@ const AudioBlock = dynamic(() => import('./codeBlock/Audio'), { ssr: false });
const ChatGuide = dynamic(() => import('./chat/Guide'), { ssr: false });
const QuestionGuide = dynamic(() => import('./chat/QuestionGuide'), { ssr: false });
const A = dynamic(() => import('./A'), { ssr: false });
function isSafeHref(href: string): boolean {
if (!href) return false;
// allow http(s), mailto, tel, relative paths, #, data:image/audio/video
return /^(https?:|mailto:|tel:|\/|#|data:(?:image|audio|video))/i.test(href.trim());
}
const SafeA = (props: any) => {
const href = props.href || '';
const safeHref = isSafeHref(href) ? href : '#';
const ALLOWED_A_ATTRS = new Set([
'href',
'target',
'rel',
'className',
'children',
'style',
'title'
]);
const safeProps = filterSafeProps(props, ALLOWED_A_ATTRS);
return (
<a
{...safeProps}
href={safeHref}
target="_blank"
rel="noopener noreferrer"
className="cursor-pointer underline !decoration-primary-700 decoration-dashed"
>
{props.children || 'Download'}
</a>
);
};
type Props = {
source?: string;
@@ -36,11 +69,9 @@ type Props = {
} & AProps;
const Markdown = (props: Props) => {
const source = props.source || '';
if (source.length < 200000) {
return <MarkdownRender {...props} />;
}
return <Box whiteSpace={'pre-wrap'}>{source}</Box>;
};
const MarkdownRender = ({
@@ -48,7 +79,6 @@ const MarkdownRender = ({
showAnimation,
isDisabled,
forbidZhFormat,
chatAuthData,
onOpenCiteModal
}: Props) => {
@@ -57,54 +87,116 @@ const MarkdownRender = ({
img: Image,
pre: RewritePre,
code: Code,
svg: SVGRenderer,
script: ScriptBlock,
a: (props: any) => (
<A
<SafeA
{...props}
showAnimation={showAnimation}
chatAuthData={chatAuthData}
onOpenCiteModal={onOpenCiteModal}
showAnimation={showAnimation}
/>
)
};
}, [chatAuthData, onOpenCiteModal, showAnimation]);
const formatSource = useMemo(() => {
if (showAnimation || forbidZhFormat) return source;
return mdTextFormat(source);
}, [forbidZhFormat, showAnimation, source]);
const urlTransform = useCallback((val: string) => {
return val;
}, []);
const urlTransform = useCallback((val: string) => val, []);
return (
<Box position={'relative'}>
<ReactMarkdown
className={`markdown ${styles.markdown}
${showAnimation ? `${formatSource ? styles.waitingAnimation : styles.animation}` : ''}
`}
remarkPlugins={[RemarkMath, [RemarkGfm, { singleTilde: false }], RemarkBreaks]}
rehypePlugins={[RehypeKatex, [RehypeExternalLinks, { target: '_blank' }]]}
components={components}
urlTransform={urlTransform}
>
{formatSource}
</ReactMarkdown>
<ErrorBoundary>
<ReactMarkdown
className={`markdown ${styles.markdown} ${
showAnimation ? `${formatSource ? styles.waitingAnimation : styles.animation}` : ''
}`}
remarkPlugins={[RemarkMath, [RemarkGfm, { singleTilde: false }], RemarkBreaks]}
rehypePlugins={[
RehypeKatex,
[RehypeExternalLinks, { target: '_blank', rel: ['noopener', 'noreferrer'] }],
RehypeRaw as any,
() => {
return (tree) => {
const iterate = (node: any) => {
if (node.type === 'element') {
// delete ref
if (node.properties?.ref) delete node.properties.ref;
// handle invalid tag name
if (!/^[a-z][a-z0-9]*$/i.test(node.tagName)) {
node.type = 'text';
node.value = `<${node.tagName}`;
}
// handle properties, filter events
if (node.properties) {
Object.keys(node.properties).forEach((key) => {
const keyLower = key.toLowerCase();
// if event property (on开头)
if (keyLower.startsWith('on')) {
const value = node.properties[key];
// if event value is not a function or contains suspicious content, delete the event
if (
typeof value === 'string' || // delete event handler in string format
value === null ||
value === undefined ||
(typeof value === 'string' &&
(value.includes('javascript:') ||
value.includes('alert') ||
value.includes('eval') ||
value.includes('Function') ||
/[\(\)\[\]\{\}]/.test(value))) // flag for executable code containing parentheses, etc.
) {
delete node.properties[key];
}
}
});
}
}
// recursive handle child nodes
if (node.children) node.children.forEach(iterate);
};
tree.children.forEach(iterate);
};
}
]}
disallowedElements={[
'iframe',
'head',
'html',
'meta',
'link',
'style',
'body',
'embed',
'object',
'param',
'applet',
'area',
'map',
'isindex'
]}
components={components}
urlTransform={urlTransform}
>
{formatSource}
</ReactMarkdown>
</ErrorBoundary>
{isDisabled && <Box position={'absolute'} top={0} right={0} left={0} bottom={0} />}
</Box>
);
};
export default React.memo(Markdown);
/* Custom dom */
function Code(e: any) {
const { className, codeBlock, children } = e;
const match = /language-(\w+)/.exec(className || '');
const codeType = match?.[1]?.toLowerCase();
const strChildren = String(children);
const Component = useMemo(() => {
if (codeType === CodeClassNameEnum.mermaid) {
return <MermaidCodeBlock code={strChildren} />;
@@ -134,22 +226,61 @@ function Code(e: any) {
if (codeType === CodeClassNameEnum.audio) {
return <AudioBlock code={strChildren} />;
}
return (
<CodeLight className={className} codeBlock={codeBlock} match={match}>
{children}
</CodeLight>
);
}, [codeType, className, codeBlock, match, children, strChildren]);
return Component;
}
function Image({ src }: { src?: string }) {
return <MdImage src={src} />;
function sanitizeImageSrc(src?: string): string | undefined {
if (!src) return undefined;
// remove leading and trailing spaces
const trimmed = src.trim();
// only allow http/https/data/blob protocols
if (/^(https?:|data:|blob:)/i.test(trimmed)) {
return trimmed;
}
// allow relative paths (not starting with javascript: or vbscript:)
if (
!/^(\w+:)/.test(trimmed) &&
!trimmed.startsWith('javascript:') &&
!trimmed.startsWith('vbscript:')
) {
return trimmed;
}
return undefined;
}
function RewritePre({ children }: any) {
const ALLOWED_IMG_ATTRS = new Set([
'alt',
'width',
'height',
'className',
'style',
'title',
'loading',
'decoding',
'crossOrigin',
'referrerPolicy'
]);
function Image({ src, ...rest }: { src?: string; [key: string]: any }) {
const safeSrc = sanitizeImageSrc(src);
if (!safeSrc) {
console.warn(`Blocked unsafe image src: ${src}`);
}
// only allow whitelist attributes, and remove all on* events
const safeProps = filterSafeProps(rest, ALLOWED_IMG_ATTRS);
return <MdImage src={safeSrc} {...safeProps} />;
}
function RewritePre({ children, ...rest }: any) {
// only allow className, style, etc. safe attributes
const ALLOWED_PRE_ATTRS = new Set(['className', 'style']);
const safeProps = filterSafeProps(rest, ALLOWED_PRE_ATTRS);
const modifiedChildren = React.Children.map(children, (child) => {
if (React.isValidElement(child)) {
// @ts-ignore
@@ -157,6 +288,146 @@ function RewritePre({ children }: any) {
}
return child;
});
return <>{modifiedChildren}</>;
return <pre {...safeProps}>{modifiedChildren}</pre>;
}
/**
* general safe attribute filter
* @param props input props object
* @param allowedAttrs allowed attribute name Set
* @param eventTypeCheck whether to check event type (e.g. onClick must be a function)
*/
export function filterSafeProps(
props: Record<string, any>,
allowedAttrs: Set<string>,
eventTypeCheck: boolean = true
) {
// dangerous protocols
const DANGEROUS_PROTOCOLS =
/^(?:\s|&nbsp;|&#160;)*(?:javascript|vbscript|data(?!:(?:image|audio|video)))/i;
// dangerous event properties (including various possible ways)
const DANGEROUS_EVENTS =
/^(?:\s|&nbsp;|&#160;)*(?:on|formaction|data-|\[\[|\{\{|xlink:|href|src|action)/i;
// complete decode function
function fullDecode(input: string): string {
if (!input) return '';
let result = input;
let lastResult = '';
// continue decoding until no more decoding can be done
while (result !== lastResult) {
lastResult = result;
try {
// HTML entity decode
result = result.replace(/&(#?[\w\d]+);/g, (_, entity) => {
try {
const txt = document.createElement('textarea');
txt.innerHTML = `&${entity};`;
return txt.value;
} catch {
return '';
}
});
// Unicode decode (\u0061 format)
result = result.replace(/(?:\\|%5C|%5c)u([0-9a-f]{4})/gi, (_, hex) =>
String.fromCharCode(parseInt(hex, 16))
);
// URL encode decode
result = result.replace(/%([0-9a-f]{2})/gi, (_, hex) =>
String.fromCharCode(parseInt(hex, 16))
);
// octal decode
result = result.replace(/\\([0-7]{3})/gi, (_, oct) =>
String.fromCharCode(parseInt(oct, 8))
);
// hexadecimal decode (\x61 format)
result = result.replace(/(?:\\|%5C|%5c)x([0-9a-f]{2})/gi, (_, hex) =>
String.fromCharCode(parseInt(hex, 16))
);
// handle whitespace and comments
result = result.replace(/(?:\s|\/\*.*?\*\/|<!--.*?-->)+/g, '');
} catch {
break;
}
}
return result.toLowerCase();
}
// check if it contains dangerous content
function containsDangerousContent(value: string): boolean {
const decoded = fullDecode(value);
return (
// check dangerous protocol
DANGEROUS_PROTOCOLS.test(decoded) ||
// check dangerous event
DANGEROUS_EVENTS.test(decoded) ||
// check inline event
/on\w+\s*=/.test(decoded) ||
// check javascript: link
/javascript\s*:/.test(decoded) ||
// check other possible injections
/<\w+/i.test(decoded) ||
/\(\s*\)/i.test(decoded) ||
/\[\s*\]/i.test(decoded) ||
/\{\s*\}/i.test(decoded)
);
}
return Object.fromEntries(
Object.entries(props).filter(([key, value]) => {
// 1. decode and check property name
const decodedKey = fullDecode(key);
// 2. intercept all event related properties
if (DANGEROUS_EVENTS.test(decodedKey)) {
return false;
}
// 3. all properties not in the whitelist are rejected
if (!allowedAttrs.has(key)) {
return false;
}
// 4. check property value
if (typeof value === 'string') {
if (containsDangerousContent(value)) {
return false;
}
} else if (typeof value === 'object' && value !== null) {
// only allow simple style objects
if (key !== 'style') {
return false;
}
// check the value of the style object
for (const styleKey in value) {
if (containsDangerousContent(String(value[styleKey]))) {
return false;
}
}
} else if (typeof value === 'function') {
// only allow specific function properties (e.g. onClick)
if (!eventTypeCheck || decodedKey !== 'onclick') {
return false;
}
}
return true;
})
);
}
const ScriptBlock = memo(({ node }: any) => {
const scriptContent = node.children[0]?.value || '';
return `<script>${scriptContent}</script>`;
});
ScriptBlock.displayName = 'ScriptBlock';

View File

@@ -0,0 +1,108 @@
import React from 'react';
import ErrorBoundary from './errorBoundry';
import { filterSafeProps } from './index';
interface SVGProps {
children?: React.ReactNode;
className?: string;
style?: React.CSSProperties;
[key: string]: any;
}
const SVG_ALLOWED_ATTRS = new Set([
'width',
'height',
'viewBox',
'fill',
'stroke',
'd',
'x',
'y',
'cx',
'cy',
'r',
'className',
'style'
]);
const SVGRenderer = ({ children, className, style, ...props }: SVGProps) => {
// filter props
const svgProps = { ...props, className, style };
const sanitizedProps = filterSafeProps(svgProps, SVG_ALLOWED_ATTRS, false);
const sanitizeSVGContent = (content: string | React.ReactNode): string => {
if (typeof content !== 'string') {
return '';
}
let cleaned = content;
cleaned = cleaned.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
cleaned = cleaned.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
cleaned = cleaned.replace(
/<foreignObject\b[^<]*(?:(?!<\/foreignObject>)<[^<]*)*<\/foreignObject>/gi,
''
);
cleaned = cleaned.replace(/\son\w+="[^"]*"/gi, '');
cleaned = cleaned.replace(/\son\w+='[^']*'/gi, '');
cleaned = cleaned.replace(/url\s*\(\s*['"]?\s*javascript:[^)]+\)/gi, '');
cleaned = cleaned.replace(/\bhref="javascript:[^"]*"/gi, '');
cleaned = cleaned.replace(/\bhref='javascript:[^']*'/gi, '');
cleaned = cleaned.replace(/\bxlink:href="javascript:[^"]*"/gi, '');
cleaned = cleaned.replace(/\bxlink:href='javascript:[^']*'/gi, '');
cleaned = cleaned.replace(/\bxmlns(:xlink)?=['"]?javascript:[^"']*['"]?/gi, '');
cleaned = cleaned.replace(/style\s*=\s*(['"])(?:(?!\1).)*javascript:.*?\1/gi, '');
cleaned = cleaned.replace(/\bdata:[^,]*?;base64,[^"')]*["')]/gi, (match) => {
return match.toLowerCase().includes('javascript') ? '' : match;
});
const ALLOWED_ATTRS = new Set([
'width',
'height',
'viewBox',
'fill',
'stroke',
'd',
'x',
'y',
'cx',
'cy',
'r',
'class',
'style'
]);
cleaned = cleaned.replace(/\s(\w+)=['"][^'"]*['"]/gi, (match, attr) => {
return ALLOWED_ATTRS.has(attr.toLowerCase()) ? match : '';
});
cleaned = cleaned.replace(/<!--[\s\S]*?-->/g, '');
return cleaned;
};
const sanitizedContent = React.Children.map(children, (child) => {
if (typeof child === 'string') {
return sanitizeSVGContent(child);
}
return child;
});
return (
<ErrorBoundary fallback={<div>Something went wrong while rendering Markdown.</div>}>
<svg
{...sanitizedProps}
className={className}
style={style}
dangerouslySetInnerHTML={
typeof children === 'string' ? { __html: sanitizeSVGContent(children) } : undefined
}
>
{typeof children !== 'string' && sanitizedContent}
</svg>
</ErrorBoundary>
);
};
export default SVGRenderer;

View File

@@ -39,6 +39,12 @@ export async function register() {
systemStartCb();
initGlobalVariables();
try {
await preLoadWorker();
} catch (error) {
console.error('Preload worker error', error);
}
// Connect to MongoDB
await connectMongo(connectionMongo, MONGO_URL);
connectMongo(connectionLogMongo, MONGO_LOG_URL);
@@ -54,12 +60,6 @@ export async function register() {
startCron();
startTrainingQueue(true);
try {
await preLoadWorker();
} catch (error) {
console.error('Preload worker error', error);
}
console.log('Init system success');
}
} catch (error) {

View File

@@ -25,16 +25,20 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import SaveButton from '../Workflow/components/SaveButton';
import PublishHistories from '../PublishHistoriesSlider';
import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
import { WorkflowStatusContext } from '../WorkflowComponents/context/workflowStatusContext';
import SaveButton from '../Workflow/components/SaveButton';
const Header = () => {
const { t } = useTranslation();
const { isPc } = useSystem();
const router = useRouter();
const { toast } = useToast();
const { toast: backSaveToast } = useToast({
containerStyle: {
mt: '60px'
}
});
const { appDetail, onSaveApp, currentTab } = useContextSelector(AppContext, (v) => v);
const isV2Workflow = appDetail?.version === 'v2';
@@ -183,6 +187,7 @@ const Header = () => {
size={'sm'}
leftIcon={<MyIcon name={'core/workflow/debug'} w={['14px', '16px']} />}
variant={'whitePrimary'}
flexShrink={0}
onClick={() => {
const data = flowData2StoreDataAndCheck();
if (data) {
@@ -211,12 +216,12 @@ const Header = () => {
onBack,
onOpenBackConfirm,
isV2Workflow,
showHistoryModal,
t,
showHistoryModal,
loading,
onClickSave,
flowData2StoreDataAndCheck,
setShowHistoryModal,
flowData2StoreDataAndCheck,
setWorkflowTestData
]);
@@ -229,10 +234,11 @@ const Header = () => {
setShowHistoryModal(false);
}}
past={past}
onSwitchTmpVersion={onSwitchTmpVersion}
onSwitchCloudVersion={onSwitchCloudVersion}
onSwitchTmpVersion={onSwitchTmpVersion}
/>
)}
<MyModal
isOpen={isOpenBackConfirm}
onClose={onCloseBackConfirm}
@@ -254,7 +260,7 @@ const Header = () => {
await onClickSave({});
onCloseBackConfirm();
onBack();
toast({
backSaveToast({
status: 'success',
title: t('app:saved_success'),
position: 'top-right'

View File

@@ -13,7 +13,7 @@ import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useContextSelector } from 'use-context-selector';
import { WorkflowContext } from '../WorkflowComponents/context';
import { WorkflowContext, type WorkflowSnapshotsType } from '../WorkflowComponents/context';
import { AppContext, TabEnum } from '../context';
import RouteTab from '../RouteTab';
import { useRouter } from 'next/router';
@@ -25,10 +25,10 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import SaveButton from './components/SaveButton';
import PublishHistories from '../PublishHistoriesSlider';
import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
import { WorkflowStatusContext } from '../WorkflowComponents/context/workflowStatusContext';
import SaveButton from '../Workflow/components/SaveButton';
const Header = () => {
const { t } = useTranslation();
@@ -187,6 +187,7 @@ const Header = () => {
size={'sm'}
leftIcon={<MyIcon name={'core/workflow/debug'} w={['14px', '16px']} />}
variant={'whitePrimary'}
flexShrink={0}
onClick={() => {
const data = flowData2StoreDataAndCheck();
if (data) {
@@ -215,12 +216,12 @@ const Header = () => {
onBack,
onOpenBackConfirm,
isV2Workflow,
showHistoryModal,
t,
showHistoryModal,
loading,
onClickSave,
flowData2StoreDataAndCheck,
setShowHistoryModal,
flowData2StoreDataAndCheck,
setWorkflowTestData
]);
@@ -228,7 +229,7 @@ const Header = () => {
<>
{Render}
{showHistoryModal && isV2Workflow && currentTab === TabEnum.appEdit && (
<PublishHistories
<PublishHistories<WorkflowSnapshotsType>
onClose={() => {
setShowHistoryModal(false);
}}

View File

@@ -43,6 +43,7 @@ const SaveButton = ({
Trigger={
<Button
size={'sm'}
flexShrink={0}
rightIcon={
<MyIcon
name={isSave ? 'core/chat/chevronUp' : 'core/chat/chevronDown'}

View File

@@ -0,0 +1,220 @@
import React, { useState, useCallback } from 'react';
import { Box, Flex, Button, IconButton, type ButtonProps, Input } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { useContextSelector } from 'use-context-selector';
import { WorkflowNodeEdgeContext } from '../../WorkflowComponents/context/workflowInitContext';
import { useReactFlow } from 'reactflow';
import { useKeyPress, useThrottleEffect } from 'ahooks';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
const SearchButton = (props: ButtonProps) => {
const { t } = useTranslation();
const setNodes = useContextSelector(WorkflowNodeEdgeContext, (state) => state.setNodes);
const { fitView } = useReactFlow();
const { isMac } = useSystem();
const [keyword, setKeyword] = useState<string>();
const [searchIndex, setSearchIndex] = useState<number>(0);
const [searchedNodeCount, setSearchedNodeCount] = useState(0);
useKeyPress(['ctrl.f', 'meta.f'], (e) => {
e.preventDefault();
e.stopPropagation();
setKeyword('');
});
useKeyPress(['esc'], (e) => {
e.preventDefault();
e.stopPropagation();
setKeyword(undefined);
});
const onSearch = useCallback(() => {
setNodes((nodes) => {
if (!keyword) {
setSearchIndex(0);
setSearchedNodeCount(0);
return nodes.map((node) => ({
...node,
data: {
...node.data,
searchedText: undefined
}
}));
}
const searchResult = nodes.filter((node) => {
const nodeName = t(node.data.name as any);
return nodeName.toLowerCase().includes(keyword.toLowerCase());
});
if (searchResult.length === 0) {
return nodes.map((node) => ({
...node,
data: {
...node.data,
searchedText: undefined
}
}));
}
setSearchedNodeCount(searchResult.length);
const searchedNode = searchResult[searchIndex] ?? searchResult[0];
if (searchedNode) {
fitView({ nodes: [searchedNode], padding: 0.4 });
}
return nodes.map((node) => ({
...node,
selected: node.id === searchedNode.id,
data: {
...node.data,
searchedText: searchResult.find((item) => item.id === node.id) ? keyword : undefined
}
}));
});
}, [keyword, searchIndex]);
useThrottleEffect(
() => {
onSearch();
},
[onSearch],
{
wait: 500
}
);
const goToNextMatch = useCallback(() => {
if (searchIndex === searchedNodeCount - 1) {
setSearchIndex(0);
} else {
setSearchIndex(searchIndex + 1);
}
}, [searchIndex, searchedNodeCount]);
const goToPreviousMatch = useCallback(() => {
if (searchIndex === 0) {
setSearchIndex(searchedNodeCount - 1);
} else {
setSearchIndex(searchIndex - 1);
}
}, [searchIndex, searchedNodeCount]);
const clearSearch = useCallback(() => {
setKeyword(undefined);
setSearchIndex(0);
setSearchedNodeCount(0);
}, []);
if (keyword === undefined) {
return (
<Box position={'absolute'} top={'72px'} left={6} zIndex={1}>
<MyTooltip label={isMac ? t('workflow:find_tip_mac') : t('workflow:find_tip')}>
<IconButton
icon={<MyIcon name="common/searchLight" w="20px" color={'#8A95A7'} />}
aria-label=""
variant="whitePrimary"
size={'mdSquare'}
borderRadius={'50%'}
bg={'white'}
_hover={{ bg: 'white', borderColor: 'primary.300' }}
boxShadow={'0px 4px 10px 0px rgba(19, 51, 107, 0.20)'}
{...props}
onClick={() => setKeyword('')}
/>
</MyTooltip>
</Box>
);
}
return (
<Flex
position="absolute"
top={3}
left="50%"
transform="translateX(-50%)"
pl={5}
pr={4}
py={4}
zIndex={1}
borderRadius={'lg'}
bg={'white'}
alignItems={'center'}
boxShadow={
'0px 20px 24px -8px rgba(19, 51, 107, 0.15), 0px 0px 1px 0px rgba(19, 51, 107, 0.15)'
}
border={'0.5px solid rgba(0, 0, 0, 0.13)'}
maxW={['90vw', '550px']}
w={'100%'}
>
<Input
flex="1 0 0"
h={8}
border={'none'}
px={0}
_focus={{
border: 'none',
boxShadow: 'none'
}}
fontSize={'16px'}
value={keyword}
placeholder={t('workflow:please_enter_node_name')}
autoFocus
onFocus={onSearch}
onChange={(e) => setKeyword(e.target.value)}
onKeyDown={(e) => {
if (e.key === 'Enter') {
e.preventDefault();
e.stopPropagation();
goToNextMatch();
}
}}
/>
<Box fontSize="sm" color="myGray.600" whiteSpace={'nowrap'} userSelect={'none'}>
{searchedNodeCount > 0
? `${searchIndex + 1} / ${searchedNodeCount}`
: t('workflow:no_match_node')}
</Box>
{/* Border */}
<Box h={5} w={'1px'} bg={'myGray.250'} ml={3} mr={2} />
<Button
size="xs"
variant="grayGhost"
px={2}
isDisabled={searchedNodeCount <= 1}
onClick={goToPreviousMatch}
>
{t('workflow:previous')}
</Button>
<Button
size="xs"
variant="grayGhost"
px={2}
isDisabled={searchedNodeCount <= 1}
onClick={goToNextMatch}
>
{t('workflow:next')}
</Button>
<Flex
ml={2}
borderRadius="sm"
_hover={{ bg: 'myGray.100' }}
p={'1'}
cursor="pointer"
onClick={clearSearch}
>
<MyIcon name="common/closeLight" w="1.2rem" />
</Flex>
</Flex>
);
};
export default React.memo(SearchButton);

View File

@@ -1,7 +1,6 @@
import React from 'react';
import ReactFlow, { type NodeProps, SelectionMode } from 'reactflow';
import { Box, IconButton, useDisclosure } from '@chakra-ui/react';
import { SmallCloseIcon } from '@chakra-ui/icons';
import { EDGE_TYPE, FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import dynamic from 'next/dynamic';
@@ -20,6 +19,8 @@ import ContextMenu from './components/ContextMenu';
import { WorkflowNodeEdgeContext, WorkflowInitContext } from '../context/workflowInitContext';
import { WorkflowEventContext } from '../context/workflowEventContext';
import NodeTemplatesPopover from './NodeTemplatesPopover';
import SearchButton from '../../Workflow/components/SearchButton';
import MyIcon from '@fastgpt/web/components/common/Icon';
const NodeSimple = dynamic(() => import('./nodes/NodeSimple'));
const nodeTypes: Record<FlowNodeTypeEnum, any> = {
@@ -113,20 +114,22 @@ const Workflow = () => {
<>
<IconButton
position={'absolute'}
top={5}
left={5}
top={6}
left={6}
size={'mdSquare'}
borderRadius={'50%'}
icon={<SmallCloseIcon fontSize={'26px'} />}
transform={isOpenTemplate ? '' : 'rotate(135deg)'}
icon={<MyIcon name="common/addLight" w={'26px'} />}
transition={'0.2s ease'}
aria-label={''}
zIndex={1}
boxShadow={'2px 2px 6px #85b1ff'}
boxShadow={
'0px 4px 10px 0px rgba(19, 51, 107, 0.20), 0px 0px 1px 0px rgba(19, 51, 107, 0.50)'
}
onClick={() => {
isOpenTemplate ? onCloseTemplate() : onOpenTemplate();
}}
/>
<SearchButton />
<NodeTemplatesModal isOpen={isOpenTemplate} onClose={onCloseTemplate} />
<NodeTemplatesPopover />
</>

View File

@@ -36,6 +36,7 @@ import MyTag from '@fastgpt/web/components/common/Tag/index';
import MySelect from '@fastgpt/web/components/common/MySelect';
import { useCreation } from 'ahooks';
import { formatToolError } from '@fastgpt/global/core/app/utils';
import HighlightText from '@fastgpt/web/components/common/String/HighlightText';
type Props = FlowNodeItemType & {
children?: React.ReactNode | React.ReactNode[] | string;
@@ -45,6 +46,7 @@ type Props = FlowNodeItemType & {
w?: string | number;
h?: string | number;
selected?: boolean;
searchedText?: string;
menuForbid?: {
debug?: boolean;
copy?: boolean;
@@ -70,6 +72,7 @@ const NodeCard = (props: Props) => {
h = 'full',
nodeId,
selected,
searchedText,
menuForbid,
isTool = false,
isError = false,
@@ -187,7 +190,12 @@ const NodeCard = (props: Props) => {
h={'24px'}
/>
<Box ml={2} fontSize={'18px'} fontWeight={'medium'} color={'myGray.900'}>
{t(name as any)}
<HighlightText
rawText={t(name as any)}
matchText={searchedText ?? ''}
mode={'bg'}
color={'#ffe82d'}
/>
</Box>
<Button
display={'none'}
@@ -280,6 +288,7 @@ const NodeCard = (props: Props) => {
nodeId,
isFolded,
avatar,
searchedText,
t,
name,
showVersion,

View File

@@ -49,7 +49,7 @@ const CustomTextInput = () => {
createStatus: 'waiting',
rawText: data.value,
sourceName: data.name,
icon: 'file/fill/manual'
icon: 'file/fill/txt'
}
]);
goToNext();

View File

@@ -138,18 +138,20 @@ async function handler(req: ApiRequestProps<ListAppBody>): Promise<AppListItemTy
})();
const limit = (() => {
if (getRecentlyChat) return 15;
if (searchKey) return 20;
return 1000;
if (searchKey) return 50;
return;
})();
const myApps = await MongoApp.find(
findAppsQuery,
'_id parentId avatar type name intro tmbId updateTime pluginData inheritPermission'
'_id parentId avatar type name intro tmbId updateTime pluginData inheritPermission',
{
limit: limit
}
)
.sort({
updateTime: -1
})
.limit(limit)
.lean();
// Add app permission and filter apps by read permission

View File

@@ -4,11 +4,11 @@ import { type FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/d
import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { MongoRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/schema';
import { NextAPI } from '@/service/middleware/entry';
import { type ApiRequestProps } from '@fastgpt/service/type/next';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { type CreateCollectionResponse } from '@/global/core/dataset/api';
import { deleteRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/controller';
async function handler(
req: ApiRequestProps<FileIdCreateDatasetCollectionParams>
@@ -52,7 +52,7 @@ async function handler(
});
// remove buffer
await MongoRawTextBuffer.deleteOne({ sourceId: fileId });
await deleteRawTextBuffer(fileId);
return {
collectionId,

View File

@@ -6,6 +6,7 @@ import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant
import { NextAPI } from '@/service/middleware/entry';
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
import { type CreateCollectionResponse } from '@/global/core/dataset/api';
import { createFileFromText } from '@fastgpt/service/common/file/gridfs/utils';
async function handler(req: NextApiRequest): CreateCollectionResponse {
const { name, text, ...body } = req.body as TextCreateDatasetCollectionParams;
@@ -18,6 +19,18 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
per: WritePermissionVal
});
// 1. Create file from text
const filename = `${name}.txt`;
const { fileId } = await createFileFromText({
bucket: 'dataset',
filename,
text,
metadata: {
teamId,
uid: tmbId
}
});
const { collectionId, insertResults } = await createCollectionAndInsertData({
dataset,
rawText: text,
@@ -25,9 +38,9 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
...body,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.virtual,
name
type: DatasetCollectionTypeEnum.file,
fileId,
name: filename
}
});

View File

@@ -11,6 +11,7 @@ import { checkTimerLock } from '@fastgpt/service/common/system/timerLock/utils';
import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants';
import { addHours } from 'date-fns';
import { getScheduleTriggerApp } from '@/service/core/app/utils';
import { clearExpiredRawTextBufferCron } from '@fastgpt/service/common/buffer/rawText/controller';
// Try to run train every minute
const setTrainingQueueCron = () => {
@@ -83,4 +84,5 @@ export const startCron = () => {
setClearTmpUploadFilesCron();
clearInvalidDataCron();
scheduleTriggerAppCron();
clearExpiredRawTextBufferCron();
};

View File

@@ -2,7 +2,7 @@ import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/sch
import { pushQAUsage } from '@/service/support/wallet/usage/push';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { createChatCompletion } from '@fastgpt/service/core/ai/config';
import type { ChatCompletionMessageParam, StreamChatType } from '@fastgpt/global/core/ai/type.d';
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d';
import { addLog } from '@fastgpt/service/common/system/log';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { replaceVariable } from '@fastgpt/global/common/string/tools';

View File

@@ -1,6 +1,6 @@
import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
import { checkTeamAIPoints } from '@fastgpt/service/support/permission/teamLimit';
import { sendOneInform } from '../support/user/inform/api';
import { sendOneInform } from '../../../support/user/inform/api';
import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller';
import { InformLevelEnum } from '@fastgpt/global/support/user/inform/constants';
@@ -18,7 +18,7 @@ export const checkTeamAiPointsAndLock = async (teamId: string) => {
templateParam: {},
teamId
});
console.log('余额不足,暂停【向量】生成任务');
console.log('余额不足,暂停训练生成任务');
await lockTrainingDataByTeamId(teamId);
} catch (error) {}
}

View File

@@ -1,5 +1,5 @@
import { generateQA } from '@/service/events/generateQA';
import { generateVector } from '@/service/events/generateVector';
import { generateQA } from '@/service/core/dataset/queues/generateQA';
import { generateVector } from '@/service/core/dataset/queues/generateVector';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { type DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';