Compare commits
8 Commits
test-openG
...
test-html
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d7a722a609 | ||
|
|
0f866fc552 | ||
|
|
05c7ba4483 | ||
|
|
fa80ce3a77 | ||
|
|
830358aa72 | ||
|
|
02b214b3ec | ||
|
|
a171c7b11c | ||
|
|
802de11363 |
@@ -132,15 +132,15 @@ services:
|
||||
# fastgpt
|
||||
sandbox:
|
||||
container_name: sandbox
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
|
||||
networks:
|
||||
- fastgpt
|
||||
restart: always
|
||||
fastgpt-mcp-server:
|
||||
container_name: fastgpt-mcp-server
|
||||
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
|
||||
ports:
|
||||
- 3005:3000
|
||||
networks:
|
||||
@@ -150,8 +150,8 @@ services:
|
||||
- FASTGPT_ENDPOINT=http://fastgpt:3000
|
||||
fastgpt:
|
||||
container_name: fastgpt
|
||||
image: ghcr.io/labring/fastgpt:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
|
||||
ports:
|
||||
- 3000:3000
|
||||
networks:
|
||||
|
||||
@@ -109,15 +109,15 @@ services:
|
||||
# fastgpt
|
||||
sandbox:
|
||||
container_name: sandbox
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
|
||||
networks:
|
||||
- fastgpt
|
||||
restart: always
|
||||
fastgpt-mcp-server:
|
||||
container_name: fastgpt-mcp-server
|
||||
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
|
||||
ports:
|
||||
- 3005:3000
|
||||
networks:
|
||||
@@ -127,8 +127,8 @@ services:
|
||||
- FASTGPT_ENDPOINT=http://fastgpt:3000
|
||||
fastgpt:
|
||||
container_name: fastgpt
|
||||
image: ghcr.io/labring/fastgpt:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
|
||||
ports:
|
||||
- 3000:3000
|
||||
networks:
|
||||
|
||||
@@ -96,15 +96,15 @@ services:
|
||||
# fastgpt
|
||||
sandbox:
|
||||
container_name: sandbox
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
|
||||
networks:
|
||||
- fastgpt
|
||||
restart: always
|
||||
fastgpt-mcp-server:
|
||||
container_name: fastgpt-mcp-server
|
||||
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
|
||||
ports:
|
||||
- 3005:3000
|
||||
networks:
|
||||
@@ -114,8 +114,8 @@ services:
|
||||
- FASTGPT_ENDPOINT=http://fastgpt:3000
|
||||
fastgpt:
|
||||
container_name: fastgpt
|
||||
image: ghcr.io/labring/fastgpt:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
|
||||
ports:
|
||||
- 3000:3000
|
||||
networks:
|
||||
|
||||
@@ -72,15 +72,15 @@ services:
|
||||
|
||||
sandbox:
|
||||
container_name: sandbox
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-sandbox:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.10-fix2 # 阿里云
|
||||
networks:
|
||||
- fastgpt
|
||||
restart: always
|
||||
fastgpt-mcp-server:
|
||||
container_name: fastgpt-mcp-server
|
||||
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt-mcp_server:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-mcp_server:v4.9.10-fix2 # 阿里云
|
||||
ports:
|
||||
- 3005:3000
|
||||
networks:
|
||||
@@ -90,8 +90,8 @@ services:
|
||||
- FASTGPT_ENDPOINT=http://fastgpt:3000
|
||||
fastgpt:
|
||||
container_name: fastgpt
|
||||
image: ghcr.io/labring/fastgpt:v4.9.10 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10 # 阿里云
|
||||
image: ghcr.io/labring/fastgpt:v4.9.10-fix2 # git
|
||||
# image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.10-fix2 # 阿里云
|
||||
ports:
|
||||
- 3000:3000
|
||||
networks:
|
||||
|
||||
@@ -15,8 +15,8 @@ weight: 790
|
||||
|
||||
### 2. 更新镜像 tag
|
||||
|
||||
- 更新 FastGPT 镜像 tag: v4.9.10
|
||||
- 更新 FastGPT 商业版镜像 tag: v4.9.10
|
||||
- 更新 FastGPT 镜像 tag: v4.9.10-fix2
|
||||
- 更新 FastGPT 商业版镜像 tag: v4.9.10-fix2
|
||||
- mcp_server 无需更新
|
||||
- Sandbox 无需更新
|
||||
- AIProxy 无需更新
|
||||
|
||||
@@ -10,12 +10,16 @@ weight: 789
|
||||
|
||||
## 🚀 新增内容
|
||||
|
||||
1. 工作流中,子流程版本控制,可选择“保持最新版本”,无需手动更新。
|
||||
1. 工作流中增加节点搜索功能。
|
||||
2. 工作流中,子流程版本控制,可选择“保持最新版本”,无需手动更新。
|
||||
|
||||
## ⚙️ 优化
|
||||
|
||||
|
||||
1. 原文缓存改用 gridfs 存储,提高上限。
|
||||
|
||||
## 🐛 修复
|
||||
|
||||
1. 工作流中,管理员声明的全局系统工具,无法进行版本管理。
|
||||
1. 工作流中,管理员声明的全局系统工具,无法进行版本管理。
|
||||
2. 工具调用节点前,有交互节点时,上下文异常。
|
||||
3. 修复备份导入,小于 1000 字时,无法分块问题。
|
||||
4. 自定义 PDF 解析,无法保存 base64 图片。
|
||||
@@ -40,5 +40,6 @@ export function getSourceNameIcon({
|
||||
export const predictDataLimitLength = (mode: TrainingModeEnum, data: any[]) => {
|
||||
if (mode === TrainingModeEnum.qa) return data.length * 20;
|
||||
if (mode === TrainingModeEnum.auto) return data.length * 5;
|
||||
if (mode === TrainingModeEnum.image) return data.length * 2;
|
||||
return data.length;
|
||||
};
|
||||
|
||||
1
packages/global/core/workflow/type/node.d.ts
vendored
1
packages/global/core/workflow/type/node.d.ts
vendored
@@ -125,6 +125,7 @@ export type FlowNodeItemType = FlowNodeTemplateType & {
|
||||
nodeId: string;
|
||||
parentNodeId?: string;
|
||||
isError?: boolean;
|
||||
searchedText?: string;
|
||||
debugResult?: {
|
||||
status: 'running' | 'success' | 'skipped' | 'failed';
|
||||
message?: string;
|
||||
|
||||
178
packages/service/common/buffer/rawText/controller.ts
Normal file
178
packages/service/common/buffer/rawText/controller.ts
Normal file
@@ -0,0 +1,178 @@
|
||||
import { retryFn } from '@fastgpt/global/common/system/utils';
|
||||
import { connectionMongo } from '../../mongo';
|
||||
import { MongoRawTextBufferSchema, bucketName } from './schema';
|
||||
import { addLog } from '../../system/log';
|
||||
import { setCron } from '../../system/cron';
|
||||
import { checkTimerLock } from '../../system/timerLock/utils';
|
||||
import { TimerIdEnum } from '../../system/timerLock/constants';
|
||||
|
||||
const getGridBucket = () => {
|
||||
return new connectionMongo.mongo.GridFSBucket(connectionMongo.connection.db!, {
|
||||
bucketName: bucketName
|
||||
});
|
||||
};
|
||||
|
||||
export const addRawTextBuffer = async ({
|
||||
sourceId,
|
||||
sourceName,
|
||||
text,
|
||||
expiredTime
|
||||
}: {
|
||||
sourceId: string;
|
||||
sourceName: string;
|
||||
text: string;
|
||||
expiredTime: Date;
|
||||
}) => {
|
||||
const gridBucket = getGridBucket();
|
||||
const metadata = {
|
||||
sourceId,
|
||||
sourceName,
|
||||
expiredTime
|
||||
};
|
||||
|
||||
const buffer = Buffer.from(text);
|
||||
|
||||
const fileSize = buffer.length;
|
||||
// 单块大小:尽可能大,但不超过 14MB,不小于128KB
|
||||
const chunkSizeBytes = (() => {
|
||||
// 计算理想块大小:文件大小 ÷ 目标块数(10)。 并且每个块需要小于 14MB
|
||||
const idealChunkSize = Math.min(Math.ceil(fileSize / 10), 14 * 1024 * 1024);
|
||||
|
||||
// 确保块大小至少为128KB
|
||||
const minChunkSize = 128 * 1024; // 128KB
|
||||
|
||||
// 取理想块大小和最小块大小中的较大值
|
||||
let chunkSize = Math.max(idealChunkSize, minChunkSize);
|
||||
|
||||
// 将块大小向上取整到最接近的64KB的倍数,使其更整齐
|
||||
chunkSize = Math.ceil(chunkSize / (64 * 1024)) * (64 * 1024);
|
||||
|
||||
return chunkSize;
|
||||
})();
|
||||
|
||||
const uploadStream = gridBucket.openUploadStream(sourceId, {
|
||||
metadata,
|
||||
chunkSizeBytes
|
||||
});
|
||||
|
||||
return retryFn(async () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
uploadStream.end(buffer);
|
||||
uploadStream.on('finish', () => {
|
||||
resolve(uploadStream.id);
|
||||
});
|
||||
uploadStream.on('error', (error) => {
|
||||
addLog.error('addRawTextBuffer error', error);
|
||||
resolve('');
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
export const getRawTextBuffer = async (sourceId: string) => {
|
||||
const gridBucket = getGridBucket();
|
||||
|
||||
return retryFn(async () => {
|
||||
const bufferData = await MongoRawTextBufferSchema.findOne(
|
||||
{
|
||||
'metadata.sourceId': sourceId
|
||||
},
|
||||
'_id metadata'
|
||||
).lean();
|
||||
if (!bufferData) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Read file content
|
||||
const downloadStream = gridBucket.openDownloadStream(bufferData._id);
|
||||
const chunks: Buffer[] = [];
|
||||
|
||||
return new Promise<{
|
||||
text: string;
|
||||
sourceName: string;
|
||||
} | null>((resolve, reject) => {
|
||||
downloadStream.on('data', (chunk) => {
|
||||
chunks.push(chunk);
|
||||
});
|
||||
|
||||
downloadStream.on('end', () => {
|
||||
const buffer = Buffer.concat(chunks);
|
||||
const text = buffer.toString('utf8');
|
||||
resolve({
|
||||
text,
|
||||
sourceName: bufferData.metadata?.sourceName || ''
|
||||
});
|
||||
});
|
||||
|
||||
downloadStream.on('error', (error) => {
|
||||
addLog.error('getRawTextBuffer error', error);
|
||||
resolve(null);
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
export const deleteRawTextBuffer = async (sourceId: string): Promise<boolean> => {
|
||||
const gridBucket = getGridBucket();
|
||||
|
||||
return retryFn(async () => {
|
||||
const buffer = await MongoRawTextBufferSchema.findOne({ 'metadata.sourceId': sourceId });
|
||||
if (!buffer) {
|
||||
return false;
|
||||
}
|
||||
|
||||
await gridBucket.delete(buffer._id);
|
||||
return true;
|
||||
});
|
||||
};
|
||||
|
||||
export const updateRawTextBufferExpiredTime = async ({
|
||||
sourceId,
|
||||
expiredTime
|
||||
}: {
|
||||
sourceId: string;
|
||||
expiredTime: Date;
|
||||
}) => {
|
||||
return retryFn(async () => {
|
||||
return MongoRawTextBufferSchema.updateOne(
|
||||
{ 'metadata.sourceId': sourceId },
|
||||
{ $set: { 'metadata.expiredTime': expiredTime } }
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
export const clearExpiredRawTextBufferCron = async () => {
|
||||
const clearExpiredRawTextBuffer = async () => {
|
||||
addLog.debug('Clear expired raw text buffer start');
|
||||
const gridBucket = getGridBucket();
|
||||
|
||||
return retryFn(async () => {
|
||||
const data = await MongoRawTextBufferSchema.find(
|
||||
{
|
||||
'metadata.expiredTime': { $lt: new Date() }
|
||||
},
|
||||
'_id'
|
||||
).lean();
|
||||
|
||||
for (const item of data) {
|
||||
await gridBucket.delete(item._id);
|
||||
}
|
||||
addLog.debug('Clear expired raw text buffer end');
|
||||
});
|
||||
};
|
||||
|
||||
setCron('*/10 * * * *', async () => {
|
||||
if (
|
||||
await checkTimerLock({
|
||||
timerId: TimerIdEnum.clearExpiredRawTextBuffer,
|
||||
lockMinuted: 9
|
||||
})
|
||||
) {
|
||||
try {
|
||||
await clearExpiredRawTextBuffer();
|
||||
} catch (error) {
|
||||
addLog.error('clearExpiredRawTextBufferCron error', error);
|
||||
}
|
||||
}
|
||||
});
|
||||
};
|
||||
@@ -1,33 +1,22 @@
|
||||
import { getMongoModel, Schema } from '../../mongo';
|
||||
import { type RawTextBufferSchemaType } from './type';
|
||||
import { getMongoModel, type Types, Schema } from '../../mongo';
|
||||
|
||||
export const collectionName = 'buffer_rawtexts';
|
||||
export const bucketName = 'buffer_rawtext';
|
||||
|
||||
const RawTextBufferSchema = new Schema({
|
||||
sourceId: {
|
||||
type: String,
|
||||
required: true
|
||||
},
|
||||
rawText: {
|
||||
type: String,
|
||||
default: ''
|
||||
},
|
||||
createTime: {
|
||||
type: Date,
|
||||
default: () => new Date()
|
||||
},
|
||||
metadata: Object
|
||||
metadata: {
|
||||
sourceId: { type: String, required: true },
|
||||
sourceName: { type: String, required: true },
|
||||
expiredTime: { type: Date, required: true }
|
||||
}
|
||||
});
|
||||
RawTextBufferSchema.index({ 'metadata.sourceId': 'hashed' });
|
||||
RawTextBufferSchema.index({ 'metadata.expiredTime': -1 });
|
||||
|
||||
try {
|
||||
RawTextBufferSchema.index({ sourceId: 1 });
|
||||
// 20 minutes
|
||||
RawTextBufferSchema.index({ createTime: 1 }, { expireAfterSeconds: 20 * 60 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
|
||||
export const MongoRawTextBuffer = getMongoModel<RawTextBufferSchemaType>(
|
||||
collectionName,
|
||||
RawTextBufferSchema
|
||||
);
|
||||
export const MongoRawTextBufferSchema = getMongoModel<{
|
||||
_id: Types.ObjectId;
|
||||
metadata: {
|
||||
sourceId: string;
|
||||
sourceName: string;
|
||||
expiredTime: Date;
|
||||
};
|
||||
}>(`${bucketName}.files`, RawTextBufferSchema);
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
export type RawTextBufferSchemaType = {
|
||||
sourceId: string;
|
||||
rawText: string;
|
||||
createTime: Date;
|
||||
metadata?: {
|
||||
filename: string;
|
||||
};
|
||||
};
|
||||
@@ -6,13 +6,13 @@ import { type DatasetFileSchema } from '@fastgpt/global/core/dataset/type';
|
||||
import { MongoChatFileSchema, MongoDatasetFileSchema } from './schema';
|
||||
import { detectFileEncoding, detectFileEncodingByPath } from '@fastgpt/global/common/file/tools';
|
||||
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
|
||||
import { MongoRawTextBuffer } from '../../buffer/rawText/schema';
|
||||
import { readRawContentByFileBuffer } from '../read/utils';
|
||||
import { gridFsStream2Buffer, stream2Encoding } from './utils';
|
||||
import { addLog } from '../../system/log';
|
||||
import { readFromSecondary } from '../../mongo/utils';
|
||||
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
|
||||
import { Readable } from 'stream';
|
||||
import { addRawTextBuffer, getRawTextBuffer } from '../../buffer/rawText/controller';
|
||||
import { addMinutes } from 'date-fns';
|
||||
|
||||
export function getGFSCollection(bucket: `${BucketNameEnum}`) {
|
||||
MongoDatasetFileSchema;
|
||||
@@ -223,15 +223,13 @@ export const readFileContentFromMongo = async ({
|
||||
rawText: string;
|
||||
filename: string;
|
||||
}> => {
|
||||
const bufferId = `${fileId}-${customPdfParse}`;
|
||||
const bufferId = `${String(fileId)}-${customPdfParse}`;
|
||||
// read buffer
|
||||
const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: bufferId }, undefined, {
|
||||
...readFromSecondary
|
||||
}).lean();
|
||||
const fileBuffer = await getRawTextBuffer(bufferId);
|
||||
if (fileBuffer) {
|
||||
return {
|
||||
rawText: fileBuffer.rawText,
|
||||
filename: fileBuffer.metadata?.filename || ''
|
||||
rawText: fileBuffer.text,
|
||||
filename: fileBuffer?.sourceName
|
||||
};
|
||||
}
|
||||
|
||||
@@ -265,16 +263,13 @@ export const readFileContentFromMongo = async ({
|
||||
}
|
||||
});
|
||||
|
||||
// < 14M
|
||||
if (fileBuffers.length < 14 * 1024 * 1024 && rawText.trim()) {
|
||||
MongoRawTextBuffer.create({
|
||||
sourceId: bufferId,
|
||||
rawText,
|
||||
metadata: {
|
||||
filename: file.filename
|
||||
}
|
||||
});
|
||||
}
|
||||
// Add buffer
|
||||
addRawTextBuffer({
|
||||
sourceId: bufferId,
|
||||
sourceName: file.filename,
|
||||
text: rawText,
|
||||
expiredTime: addMinutes(new Date(), 20)
|
||||
});
|
||||
|
||||
return {
|
||||
rawText,
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import { Schema, getMongoModel } from '../../mongo';
|
||||
|
||||
const DatasetFileSchema = new Schema({});
|
||||
const ChatFileSchema = new Schema({});
|
||||
const DatasetFileSchema = new Schema({
|
||||
metadata: Object
|
||||
});
|
||||
const ChatFileSchema = new Schema({
|
||||
metadata: Object
|
||||
});
|
||||
|
||||
try {
|
||||
DatasetFileSchema.index({ uploadDate: -1 });
|
||||
DatasetFileSchema.index({ uploadDate: -1 });
|
||||
|
||||
ChatFileSchema.index({ uploadDate: -1 });
|
||||
ChatFileSchema.index({ 'metadata.chatId': 1 });
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
ChatFileSchema.index({ uploadDate: -1 });
|
||||
ChatFileSchema.index({ 'metadata.chatId': 1 });
|
||||
|
||||
export const MongoDatasetFileSchema = getMongoModel('dataset.files', DatasetFileSchema);
|
||||
export const MongoChatFileSchema = getMongoModel('chat.files', ChatFileSchema);
|
||||
|
||||
@@ -1,5 +1,57 @@
|
||||
import { detectFileEncoding } from '@fastgpt/global/common/file/tools';
|
||||
import { PassThrough } from 'stream';
|
||||
import { getGridBucket } from './controller';
|
||||
import { type BucketNameEnum } from '@fastgpt/global/common/file/constants';
|
||||
import { retryFn } from '@fastgpt/global/common/system/utils';
|
||||
|
||||
export const createFileFromText = async ({
|
||||
bucket,
|
||||
filename,
|
||||
text,
|
||||
metadata
|
||||
}: {
|
||||
bucket: `${BucketNameEnum}`;
|
||||
filename: string;
|
||||
text: string;
|
||||
metadata: Record<string, any>;
|
||||
}) => {
|
||||
const gridBucket = getGridBucket(bucket);
|
||||
|
||||
const buffer = Buffer.from(text);
|
||||
|
||||
const fileSize = buffer.length;
|
||||
// 单块大小:尽可能大,但不超过 14MB,不小于128KB
|
||||
const chunkSizeBytes = (() => {
|
||||
// 计算理想块大小:文件大小 ÷ 目标块数(10)。 并且每个块需要小于 14MB
|
||||
const idealChunkSize = Math.min(Math.ceil(fileSize / 10), 14 * 1024 * 1024);
|
||||
|
||||
// 确保块大小至少为128KB
|
||||
const minChunkSize = 128 * 1024; // 128KB
|
||||
|
||||
// 取理想块大小和最小块大小中的较大值
|
||||
let chunkSize = Math.max(idealChunkSize, minChunkSize);
|
||||
|
||||
// 将块大小向上取整到最接近的64KB的倍数,使其更整齐
|
||||
chunkSize = Math.ceil(chunkSize / (64 * 1024)) * (64 * 1024);
|
||||
|
||||
return chunkSize;
|
||||
})();
|
||||
|
||||
const uploadStream = gridBucket.openUploadStream(filename, {
|
||||
metadata,
|
||||
chunkSizeBytes
|
||||
});
|
||||
|
||||
return retryFn(async () => {
|
||||
return new Promise<{ fileId: string }>((resolve, reject) => {
|
||||
uploadStream.end(buffer);
|
||||
uploadStream.on('finish', () => {
|
||||
resolve({ fileId: String(uploadStream.id) });
|
||||
});
|
||||
uploadStream.on('error', reject);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
export const gridFsStream2Buffer = (stream: NodeJS.ReadableStream) => {
|
||||
return new Promise<Buffer>((resolve, reject) => {
|
||||
|
||||
@@ -110,7 +110,7 @@ export const readRawContentByFileBuffer = async ({
|
||||
|
||||
return {
|
||||
rawText: text,
|
||||
formatText: rawText,
|
||||
formatText: text,
|
||||
imageList
|
||||
};
|
||||
};
|
||||
|
||||
@@ -5,7 +5,8 @@ export enum TimerIdEnum {
|
||||
clearExpiredSubPlan = 'clearExpiredSubPlan',
|
||||
updateStandardPlan = 'updateStandardPlan',
|
||||
scheduleTriggerApp = 'scheduleTriggerApp',
|
||||
notification = 'notification'
|
||||
notification = 'notification',
|
||||
clearExpiredRawTextBuffer = 'clearExpiredRawTextBuffer'
|
||||
}
|
||||
|
||||
export enum LockNotificationEnum {
|
||||
|
||||
@@ -77,7 +77,10 @@ export const createCollectionAndInsertData = async ({
|
||||
const chunkSplitter = computeChunkSplitter(createCollectionParams);
|
||||
const paragraphChunkDeep = computeParagraphChunkDeep(createCollectionParams);
|
||||
|
||||
if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
|
||||
if (
|
||||
trainingType === DatasetCollectionDataProcessModeEnum.qa ||
|
||||
trainingType === DatasetCollectionDataProcessModeEnum.backup
|
||||
) {
|
||||
delete createCollectionParams.chunkTriggerType;
|
||||
delete createCollectionParams.chunkTriggerMinSize;
|
||||
delete createCollectionParams.dataEnhanceCollectionName;
|
||||
|
||||
@@ -218,6 +218,10 @@ export const rawText2Chunks = ({
|
||||
};
|
||||
};
|
||||
|
||||
if (backupParse) {
|
||||
return parseDatasetBackup2Chunks(rawText).chunks;
|
||||
}
|
||||
|
||||
// Chunk condition
|
||||
// 1. 选择最大值条件,只有超过了最大值(默认为模型的最大值*0.7),才会触发分块
|
||||
if (chunkTriggerType === ChunkTriggerConfigTypeEnum.maxSize) {
|
||||
@@ -240,10 +244,6 @@ export const rawText2Chunks = ({
|
||||
}
|
||||
}
|
||||
|
||||
if (backupParse) {
|
||||
return parseDatasetBackup2Chunks(rawText).chunks;
|
||||
}
|
||||
|
||||
const { chunks } = splitText2Chunks({
|
||||
text: rawText,
|
||||
chunkSize,
|
||||
|
||||
@@ -86,7 +86,6 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
|
||||
});
|
||||
|
||||
// Check interactive entry
|
||||
const interactiveResponse = lastInteractive;
|
||||
props.node.isEntry = false;
|
||||
const hasReadFilesTool = toolNodes.some(
|
||||
(item) => item.flowNodeType === FlowNodeTypeEnum.readFiles
|
||||
@@ -143,7 +142,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
|
||||
})
|
||||
}
|
||||
];
|
||||
if (interactiveResponse) {
|
||||
if (lastInteractive && isEntry) {
|
||||
return value.slice(0, -2);
|
||||
}
|
||||
return value;
|
||||
@@ -183,7 +182,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
|
||||
toolModel,
|
||||
maxRunToolTimes: 30,
|
||||
messages: adaptMessages,
|
||||
interactiveEntryToolParams: interactiveResponse?.toolParams
|
||||
interactiveEntryToolParams: lastInteractive?.toolParams
|
||||
});
|
||||
}
|
||||
if (toolModel.functionCall) {
|
||||
@@ -194,7 +193,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
|
||||
toolNodes,
|
||||
toolModel,
|
||||
messages: adaptMessages,
|
||||
interactiveEntryToolParams: interactiveResponse?.toolParams
|
||||
interactiveEntryToolParams: lastInteractive?.toolParams
|
||||
});
|
||||
}
|
||||
|
||||
@@ -224,7 +223,7 @@ export const dispatchRunTools = async (props: DispatchToolModuleProps): Promise<
|
||||
toolNodes,
|
||||
toolModel,
|
||||
messages: adaptMessages,
|
||||
interactiveEntryToolParams: interactiveResponse?.toolParams
|
||||
interactiveEntryToolParams: lastInteractive?.toolParams
|
||||
});
|
||||
})();
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ import type {
|
||||
SystemVariablesType
|
||||
} from '@fastgpt/global/core/workflow/runtime/type';
|
||||
import type { RuntimeNodeItemType } from '@fastgpt/global/core/workflow/runtime/type.d';
|
||||
import type { FlowNodeOutputItemType } from '@fastgpt/global/core/workflow/type/io.d';
|
||||
import type {
|
||||
AIChatItemValueItemType,
|
||||
ChatHistoryItemResType,
|
||||
|
||||
@@ -17,6 +17,7 @@ import { chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
|
||||
import { getPluginRunUserQuery } from '@fastgpt/global/core/workflow/utils';
|
||||
import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
|
||||
import type { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
|
||||
import { getUserChatInfoAndAuthTeamPoints } from '../../../../support/permission/auth/team';
|
||||
|
||||
type RunPluginProps = ModuleDispatchProps<{
|
||||
[NodeInputKeyEnum.forbidStream]?: boolean;
|
||||
@@ -73,9 +74,11 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
|
||||
};
|
||||
});
|
||||
|
||||
const { externalProvider } = await getUserChatInfoAndAuthTeamPoints(runningAppInfo.tmbId);
|
||||
const runtimeVariables = {
|
||||
...filterSystemVariables(props.variables),
|
||||
appId: String(plugin.id)
|
||||
appId: String(plugin.id),
|
||||
...(externalProvider ? externalProvider.externalWorkflowVariables : {})
|
||||
};
|
||||
const { flowResponses, flowUsages, assistantResponses, runTimes } = await dispatchWorkFlow({
|
||||
...props,
|
||||
|
||||
@@ -20,6 +20,7 @@ import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { getAppVersionById } from '../../../app/version/controller';
|
||||
import { parseUrlToFileType } from '@fastgpt/global/common/file/tools';
|
||||
import { type ChildrenInteractive } from '@fastgpt/global/core/workflow/template/system/interactive/type';
|
||||
import { getUserChatInfoAndAuthTeamPoints } from '../../../../support/permission/auth/team';
|
||||
|
||||
type Props = ModuleDispatchProps<{
|
||||
[NodeInputKeyEnum.userChatInput]: string;
|
||||
@@ -97,11 +98,13 @@ export const dispatchRunAppNode = async (props: Props): Promise<Response> => {
|
||||
|
||||
// Rewrite children app variables
|
||||
const systemVariables = filterSystemVariables(variables);
|
||||
const { externalProvider } = await getUserChatInfoAndAuthTeamPoints(appData.tmbId);
|
||||
const childrenRunVariables = {
|
||||
...systemVariables,
|
||||
...childrenAppVariables,
|
||||
histories: chatHistories,
|
||||
appId: String(appData._id)
|
||||
appId: String(appData._id),
|
||||
...(externalProvider ? externalProvider.externalWorkflowVariables : {})
|
||||
};
|
||||
|
||||
const childrenInteractive =
|
||||
|
||||
@@ -5,8 +5,6 @@ import { NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
|
||||
import { type DispatchNodeResultType } from '@fastgpt/global/core/workflow/runtime/type';
|
||||
import axios from 'axios';
|
||||
import { serverRequestBaseUrl } from '../../../../common/api/serverRequest';
|
||||
import { MongoRawTextBuffer } from '../../../../common/buffer/rawText/schema';
|
||||
import { readFromSecondary } from '../../../../common/mongo/utils';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import { detectFileEncoding, parseUrlToFileType } from '@fastgpt/global/common/file/tools';
|
||||
import { readRawContentByFileBuffer } from '../../../../common/file/read/utils';
|
||||
@@ -14,6 +12,8 @@ import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
|
||||
import { type ChatItemType, type UserChatItemValueItemType } from '@fastgpt/global/core/chat/type';
|
||||
import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
|
||||
import { addLog } from '../../../../common/system/log';
|
||||
import { addRawTextBuffer, getRawTextBuffer } from '../../../../common/buffer/rawText/controller';
|
||||
import { addMinutes } from 'date-fns';
|
||||
|
||||
type Props = ModuleDispatchProps<{
|
||||
[NodeInputKeyEnum.fileUrlList]: string[];
|
||||
@@ -158,14 +158,12 @@ export const getFileContentFromLinks = async ({
|
||||
parseUrlList
|
||||
.map(async (url) => {
|
||||
// Get from buffer
|
||||
const fileBuffer = await MongoRawTextBuffer.findOne({ sourceId: url }, undefined, {
|
||||
...readFromSecondary
|
||||
}).lean();
|
||||
const fileBuffer = await getRawTextBuffer(url);
|
||||
if (fileBuffer) {
|
||||
return formatResponseObject({
|
||||
filename: fileBuffer.metadata?.filename || url,
|
||||
filename: fileBuffer.sourceName || url,
|
||||
url,
|
||||
content: fileBuffer.rawText
|
||||
content: fileBuffer.text
|
||||
});
|
||||
}
|
||||
|
||||
@@ -220,17 +218,12 @@ export const getFileContentFromLinks = async ({
|
||||
});
|
||||
|
||||
// Add to buffer
|
||||
try {
|
||||
if (buffer.length < 14 * 1024 * 1024 && rawText.trim()) {
|
||||
MongoRawTextBuffer.create({
|
||||
sourceId: url,
|
||||
rawText,
|
||||
metadata: {
|
||||
filename: filename
|
||||
}
|
||||
});
|
||||
}
|
||||
} catch (error) {}
|
||||
addRawTextBuffer({
|
||||
sourceId: url,
|
||||
sourceName: filename,
|
||||
text: rawText,
|
||||
expiredTime: addMinutes(new Date(), 20)
|
||||
});
|
||||
|
||||
return formatResponseObject({ filename, url, content: rawText });
|
||||
} catch (error) {
|
||||
|
||||
@@ -1,17 +1,26 @@
|
||||
import { Box } from '@chakra-ui/react';
|
||||
import React from 'react';
|
||||
import React, { useMemo } from 'react';
|
||||
|
||||
const HighlightText = ({
|
||||
rawText,
|
||||
matchText,
|
||||
color = 'primary.600'
|
||||
color = 'primary.600',
|
||||
mode = 'text'
|
||||
}: {
|
||||
rawText: string;
|
||||
matchText: string;
|
||||
color?: string;
|
||||
mode?: 'text' | 'bg';
|
||||
}) => {
|
||||
const regex = new RegExp(`(${matchText})`, 'gi');
|
||||
const parts = rawText.split(regex);
|
||||
const { parts } = useMemo(() => {
|
||||
const regx = new RegExp(`(${matchText})`, 'gi');
|
||||
const parts = rawText.split(regx);
|
||||
|
||||
return {
|
||||
regx,
|
||||
parts
|
||||
};
|
||||
}, [rawText, matchText]);
|
||||
|
||||
return (
|
||||
<Box>
|
||||
@@ -28,7 +37,17 @@ const HighlightText = ({
|
||||
}
|
||||
|
||||
return (
|
||||
<Box as="span" key={index} color={highLight ? color : 'inherit'}>
|
||||
<Box
|
||||
as="span"
|
||||
key={index}
|
||||
{...(mode === 'bg'
|
||||
? {
|
||||
bg: highLight ? color : 'transparent'
|
||||
}
|
||||
: {
|
||||
color: highLight ? color : 'inherit'
|
||||
})}
|
||||
>
|
||||
{part}
|
||||
</Box>
|
||||
);
|
||||
@@ -37,4 +56,4 @@ const HighlightText = ({
|
||||
);
|
||||
};
|
||||
|
||||
export default HighlightText;
|
||||
export default React.memo(HighlightText);
|
||||
|
||||
@@ -3,6 +3,8 @@ import { useContextSelector } from 'use-context-selector';
|
||||
|
||||
export const useSystem = () => {
|
||||
const isPc = useContextSelector(useSystemStoreContext, (state) => state.isPc);
|
||||
const isMac =
|
||||
typeof window !== 'undefined' && window.navigator.userAgent.toLocaleLowerCase().includes('mac');
|
||||
|
||||
return { isPc };
|
||||
return { isPc, isMac };
|
||||
};
|
||||
|
||||
@@ -63,6 +63,8 @@
|
||||
"field_required": "Required",
|
||||
"field_used_as_tool_input": "Used as Tool Call Parameter",
|
||||
"filter_description": "Currently supports filtering by tags and creation time. Fill in the format as follows:\n{\n \"tags\": {\n \"$and\": [\"Tag 1\",\"Tag 2\"],\n \"$or\": [\"When there are $and tags, and is effective, or is not effective\"]\n },\n \"createTime\": {\n \"$gte\": \"YYYY-MM-DD HH:mm format, collection creation time greater than this time\",\n \"$lte\": \"YYYY-MM-DD HH:mm format, collection creation time less than this time, can be used with $gte\"\n }\n}",
|
||||
"find_tip": "Find node ctrl f",
|
||||
"find_tip_mac": "Find node ⌘ f",
|
||||
"foldAll": "Collapse all",
|
||||
"form_input_result": "User complete input result",
|
||||
"form_input_result_tip": "an object containing the complete result",
|
||||
@@ -123,18 +125,23 @@
|
||||
"max_tokens": "Maximum Tokens",
|
||||
"mouse_priority": "Mouse first\n- Press the left button to drag the canvas\n- Hold down shift and left click to select batches",
|
||||
"new_context": "New Context",
|
||||
"next": "Next",
|
||||
"no_match_node": "No results",
|
||||
"no_node_found": "No node was not found",
|
||||
"not_contains": "Does Not Contain",
|
||||
"only_the_reference_type_is_supported": "Only reference type is supported",
|
||||
"optional_value_type": "Optional Value Type",
|
||||
"optional_value_type_tip": "You can specify one or more data types. When dynamically adding fields, users can only select the configured types.",
|
||||
"pan_priority": "Touchpad first\n- Click to batch select\n- Move the canvas with two fingers",
|
||||
"pass_returned_object_as_output_to_next_nodes": "Pass the object returned in the code as output to the next nodes. The variable name needs to correspond to the return key.",
|
||||
"please_enter_node_name": "Enter the node name",
|
||||
"plugin.Instruction_Tip": "You can configure an instruction to explain the purpose of the plugin. This instruction will be displayed each time the plugin is used. Supports standard Markdown syntax.",
|
||||
"plugin.Instructions": "Instructions",
|
||||
"plugin.global_file_input": "File links (deprecated)",
|
||||
"plugin_file_abandon_tip": "Plugin global file upload has been deprecated, please adjust it as soon as possible. \nRelated functions can be achieved through plug-in input and adding image type input.",
|
||||
"plugin_input": "Plugin Input",
|
||||
"plugin_output_tool": "When the plug-in is executed as a tool, whether this field responds as a result of the tool",
|
||||
"previous": "Previous",
|
||||
"question_classification": "Classify",
|
||||
"question_optimization": "Query extension",
|
||||
"quote_content_placeholder": "The structure of the reference content can be customized to better suit different scenarios. \nSome variables can be used for template configuration\n\n{{q}} - main content\n\n{{a}} - auxiliary data\n\n{{source}} - source name\n\n{{sourceId}} - source ID\n\n{{index}} - nth reference",
|
||||
@@ -177,9 +184,9 @@
|
||||
"text_content_extraction": "Text Extract",
|
||||
"text_to_extract": "Text to Extract",
|
||||
"these_variables_will_be_input_parameters_for_code_execution": "These variables will be input parameters for code execution",
|
||||
"tool.tool_result": "Tool operation results",
|
||||
"to_add_node": "to add",
|
||||
"to_connect_node": "to connect",
|
||||
"tool.tool_result": "Tool operation results",
|
||||
"tool_call_termination": "Stop ToolCall",
|
||||
"tool_custom_field": "Custom Tool",
|
||||
"tool_field": " Tool Field Parameter Configuration",
|
||||
|
||||
@@ -63,6 +63,8 @@
|
||||
"field_required": "必填",
|
||||
"field_used_as_tool_input": "作为工具调用参数",
|
||||
"filter_description": "目前支持标签和创建时间过滤,需按照以下格式填写:\n{\n \"tags\": {\n \"$and\": [\"标签 1\",\"标签 2\"],\n \"$or\": [\"有 $and 标签时,and 生效,or 不生效\"]\n },\n \"createTime\": {\n \"$gte\": \"YYYY-MM-DD HH:mm 格式即可,集合的创建时间大于该时间\",\n \"$lte\": \"YYYY-MM-DD HH:mm 格式即可,集合的创建时间小于该时间,可和 $gte 共同使用\"\n }\n}",
|
||||
"find_tip": "查找节点 ctrl f",
|
||||
"find_tip_mac": "查找节点 ⌘ f",
|
||||
"foldAll": "全部折叠",
|
||||
"form_input_result": "用户完整输入结果",
|
||||
"form_input_result_tip": "一个包含完整结果的对象",
|
||||
@@ -123,18 +125,23 @@
|
||||
"max_tokens": "最大 Tokens",
|
||||
"mouse_priority": "鼠标优先\n- 左键按下后可拖动画布\n- 按住 shift 后左键可批量选择",
|
||||
"new_context": "新的上下文",
|
||||
"next": "下一个",
|
||||
"no_match_node": "无结果",
|
||||
"no_node_found": "未搜索到节点",
|
||||
"not_contains": "不包含",
|
||||
"only_the_reference_type_is_supported": "仅支持引用类型",
|
||||
"optional_value_type": "可选的数据类型",
|
||||
"optional_value_type_tip": "可以指定 1 个或多个数据类型,用户在动态添加字段时,仅可选择配置的类型",
|
||||
"pan_priority": "触摸板优先\n- 单击批量选择\n- 双指移动画布",
|
||||
"pass_returned_object_as_output_to_next_nodes": "将代码中 return 的对象作为输出,传递给后续的节点。变量名需要对应 return 的 key",
|
||||
"please_enter_node_name": "请输入节点名称",
|
||||
"plugin.Instruction_Tip": "可以配置一段说明,以解释该插件的用途。每次使用插件前,会显示该段说明。支持标准 Markdown 语法。",
|
||||
"plugin.Instructions": "使用说明",
|
||||
"plugin.global_file_input": "文件链接(弃用)",
|
||||
"plugin_file_abandon_tip": "插件全局文件上传已弃用,请尽快调整。可以通过插件输入,添加图片类型输入来实现相关功能。",
|
||||
"plugin_input": "插件输入",
|
||||
"plugin_output_tool": "插件作为工具执行时,该字段是否作为工具响应结果",
|
||||
"previous": "上一个",
|
||||
"question_classification": "问题分类",
|
||||
"question_optimization": "问题优化",
|
||||
"quote_content_placeholder": "可以自定义引用内容的结构,以更好的适配不同场景。可以使用一些变量来进行模板配置\n{{q}} - 主要内容\n{{a}} - 辅助数据\n{{source}} - 来源名\n{{sourceId}} - 来源ID\n{{index}} - 第 n 个引用",
|
||||
|
||||
@@ -63,6 +63,8 @@
|
||||
"field_required": "必填",
|
||||
"field_used_as_tool_input": "作為工具呼叫參數",
|
||||
"filter_description": "目前支援標籤和建立時間篩選,需按照以下格式填寫:\n{\n \"tags\": {\n \"$and\": [\"標籤 1\",\"標籤 2\"],\n \"$or\": [\"當有 $and 標籤時,$and 才會生效,$or 不會生效\"]\n },\n \"createTime\": {\n \"$gte\": \"YYYY-MM-DD HH:mm 格式,資料集的建立時間大於這個時間\",\n \"$lte\": \"YYYY-MM-DD HH:mm 格式,資料集的建立時間小於這個時間,可以和 $gte 一起使用\"\n }\n}",
|
||||
"find_tip": "查找節點 ctrl f",
|
||||
"find_tip_mac": "查找節點 ⌘ f",
|
||||
"foldAll": "全部折疊",
|
||||
"form_input_result": "使用者完整輸入結果",
|
||||
"form_input_result_tip": "一個包含完整結果的物件",
|
||||
@@ -123,18 +125,23 @@
|
||||
"max_tokens": "最大 Token 數",
|
||||
"mouse_priority": "滑鼠優先\n- 按下左鍵拖曳畫布\n- 按住 Shift 鍵並點選左鍵可批次選取",
|
||||
"new_context": "新的脈絡",
|
||||
"next": "下一個",
|
||||
"no_match_node": "無結果",
|
||||
"no_node_found": "未搜索到節點",
|
||||
"not_contains": "不包含",
|
||||
"only_the_reference_type_is_supported": "僅支援引用類型",
|
||||
"optional_value_type": "可選的資料類型",
|
||||
"optional_value_type_tip": "可以指定一或多個資料類型,使用者在動態新增欄位時,只能選擇已設定的類型",
|
||||
"pan_priority": "觸控板優先\n- 點選可批次選取\n- 使用兩指移動畫布",
|
||||
"pass_returned_object_as_output_to_next_nodes": "將程式碼中 return 的物件作為輸出,傳遞給後續的節點。變數名稱需要對應 return 的鍵值",
|
||||
"please_enter_node_name": "請輸入節點名稱",
|
||||
"plugin.Instruction_Tip": "您可以設定一段說明來解釋這個外掛程式的用途。每次使用外掛程式前,都會顯示這段說明。支援標準 Markdown 語法。",
|
||||
"plugin.Instructions": "使用說明",
|
||||
"plugin.global_file_input": "檔案連結(已淘汰)",
|
||||
"plugin_file_abandon_tip": "外掛程式全域檔案上傳功能已淘汰,請儘速調整。您可以透過外掛程式輸入,新增圖片類型輸入來達成相關功能。",
|
||||
"plugin_input": "外掛程式輸入",
|
||||
"plugin_output_tool": "外掛程式作為工具執行時,這個欄位是否作為工具的回應結果",
|
||||
"previous": "上一個",
|
||||
"question_classification": "問題分類",
|
||||
"question_optimization": "問題最佳化",
|
||||
"quote_content_placeholder": "可以自訂引用內容的結構,以便更好地適應不同場景。可以使用一些變數來設定範本\n{{q}} - 主要內容\n{{a}} - 輔助資料\n{{source}} - 來源名稱\n{{sourceId}} - 來源 ID\n{{index}} - 第 n 個引用",
|
||||
@@ -177,9 +184,9 @@
|
||||
"text_content_extraction": "文字內容擷取",
|
||||
"text_to_extract": "要擷取的文字",
|
||||
"these_variables_will_be_input_parameters_for_code_execution": "這些變數會作為程式碼執行的輸入參數",
|
||||
"tool.tool_result": "工具運行結果",
|
||||
"to_add_node": "添加節點",
|
||||
"to_connect_node": "連接節點",
|
||||
"tool.tool_result": "工具運行結果",
|
||||
"tool_call_termination": "工具呼叫終止",
|
||||
"tool_custom_field": "自訂工具變數",
|
||||
"tool_field": "工具參數設定",
|
||||
|
||||
68
pnpm-lock.yaml
generated
68
pnpm-lock.yaml
generated
@@ -46,7 +46,7 @@ importers:
|
||||
version: 10.1.4(socks@2.8.4)
|
||||
next-i18next:
|
||||
specifier: 15.4.2
|
||||
version: 15.4.2(i18next@23.16.8)(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
|
||||
version: 15.4.2(i18next@23.16.8)(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
|
||||
prettier:
|
||||
specifier: 3.2.4
|
||||
version: 3.2.4
|
||||
@@ -343,7 +343,7 @@ importers:
|
||||
version: 2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)
|
||||
'@chakra-ui/next-js':
|
||||
specifier: 2.4.2
|
||||
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
|
||||
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
|
||||
'@chakra-ui/react':
|
||||
specifier: 2.10.7
|
||||
version: 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
|
||||
@@ -406,7 +406,7 @@ importers:
|
||||
version: 4.17.21
|
||||
next-i18next:
|
||||
specifier: 15.4.2
|
||||
version: 15.4.2(i18next@23.16.8)(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
|
||||
version: 15.4.2(i18next@23.16.8)(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
|
||||
papaparse:
|
||||
specifier: ^5.4.1
|
||||
version: 5.4.1
|
||||
@@ -467,7 +467,7 @@ importers:
|
||||
version: 2.1.1(@chakra-ui/system@2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1))(react@18.3.1)
|
||||
'@chakra-ui/next-js':
|
||||
specifier: 2.4.2
|
||||
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
|
||||
version: 2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)
|
||||
'@chakra-ui/react':
|
||||
specifier: 2.10.7
|
||||
version: 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
|
||||
@@ -569,7 +569,7 @@ importers:
|
||||
version: 14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1)
|
||||
next-i18next:
|
||||
specifier: 15.4.2
|
||||
version: 15.4.2(i18next@23.16.8)(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
|
||||
version: 15.4.2(i18next@23.16.8)(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1)
|
||||
nprogress:
|
||||
specifier: ^0.2.0
|
||||
version: 0.2.0
|
||||
@@ -612,6 +612,9 @@ importers:
|
||||
rehype-katex:
|
||||
specifier: ^7.0.0
|
||||
version: 7.0.1
|
||||
rehype-raw:
|
||||
specifier: ^7.0.0
|
||||
version: 7.0.0
|
||||
remark-breaks:
|
||||
specifier: ^4.0.0
|
||||
version: 4.0.0
|
||||
@@ -5879,9 +5882,15 @@ packages:
|
||||
hast-util-parse-selector@4.0.0:
|
||||
resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==}
|
||||
|
||||
hast-util-raw@9.1.0:
|
||||
resolution: {integrity: sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==}
|
||||
|
||||
hast-util-to-jsx-runtime@2.3.6:
|
||||
resolution: {integrity: sha512-zl6s8LwNyo1P9uw+XJGvZtdFF1GdAkOg8ujOw+4Pyb76874fLps4ueHXDhXWdk6YHQ6OgUtinliG7RsYvCbbBg==}
|
||||
|
||||
hast-util-to-parse5@8.0.0:
|
||||
resolution: {integrity: sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw==}
|
||||
|
||||
hast-util-to-text@4.0.2:
|
||||
resolution: {integrity: sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==}
|
||||
|
||||
@@ -5919,6 +5928,9 @@ packages:
|
||||
html-url-attributes@3.0.1:
|
||||
resolution: {integrity: sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==}
|
||||
|
||||
html-void-elements@3.0.0:
|
||||
resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==}
|
||||
|
||||
htmlparser2@8.0.2:
|
||||
resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==}
|
||||
|
||||
@@ -8026,6 +8038,9 @@ packages:
|
||||
property-information@5.6.0:
|
||||
resolution: {integrity: sha512-YUHSPk+A30YPv+0Qf8i9Mbfe/C0hdPXk1s1jPVToV8pk8BQtpw10ct89Eo7OWkutrwqvT0eicAxlOg3dOAu8JA==}
|
||||
|
||||
property-information@6.5.0:
|
||||
resolution: {integrity: sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig==}
|
||||
|
||||
property-information@7.0.0:
|
||||
resolution: {integrity: sha512-7D/qOz/+Y4X/rzSB6jKxKUsQnphO046ei8qxG59mtM3RG3DHgTK81HrxrmoDVINJb8NKT5ZsRbwHvQ6B68Iyhg==}
|
||||
|
||||
@@ -8361,6 +8376,9 @@ packages:
|
||||
rehype-katex@7.0.1:
|
||||
resolution: {integrity: sha512-OiM2wrZ/wuhKkigASodFoo8wimG3H12LWQaH8qSPVJn9apWKFSH3YOCtbKpBorTVw/eI7cuT21XBbvwEswbIOA==}
|
||||
|
||||
rehype-raw@7.0.0:
|
||||
resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==}
|
||||
|
||||
remark-breaks@4.0.0:
|
||||
resolution: {integrity: sha512-IjEjJOkH4FuJvHZVIW0QCDWxcG96kCq7An/KVH2NfJe6rKZU2AsHeB3OEjPNRxi4QC34Xdx7I2KGYn6IpT7gxQ==}
|
||||
|
||||
@@ -10969,7 +10987,7 @@ snapshots:
|
||||
'@chakra-ui/system': 2.6.1(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(react@18.3.1)
|
||||
react: 18.3.1
|
||||
|
||||
'@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
|
||||
'@chakra-ui/next-js@2.4.2(@chakra-ui/react@2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react@18.3.1)':
|
||||
dependencies:
|
||||
'@chakra-ui/react': 2.10.7(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@emotion/styled@11.11.0(@emotion/react@11.11.1(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(react@18.3.1))(@types/react@18.3.1)(framer-motion@9.1.7(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
|
||||
'@emotion/cache': 11.14.0
|
||||
@@ -16108,6 +16126,22 @@ snapshots:
|
||||
dependencies:
|
||||
'@types/hast': 3.0.4
|
||||
|
||||
hast-util-raw@9.1.0:
|
||||
dependencies:
|
||||
'@types/hast': 3.0.4
|
||||
'@types/unist': 3.0.3
|
||||
'@ungap/structured-clone': 1.3.0
|
||||
hast-util-from-parse5: 8.0.3
|
||||
hast-util-to-parse5: 8.0.0
|
||||
html-void-elements: 3.0.0
|
||||
mdast-util-to-hast: 13.2.0
|
||||
parse5: 7.2.1
|
||||
unist-util-position: 5.0.0
|
||||
unist-util-visit: 5.0.0
|
||||
vfile: 6.0.3
|
||||
web-namespaces: 2.0.1
|
||||
zwitch: 2.0.4
|
||||
|
||||
hast-util-to-jsx-runtime@2.3.6:
|
||||
dependencies:
|
||||
'@types/estree': 1.0.6
|
||||
@@ -16128,6 +16162,16 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
hast-util-to-parse5@8.0.0:
|
||||
dependencies:
|
||||
'@types/hast': 3.0.4
|
||||
comma-separated-tokens: 2.0.3
|
||||
devlop: 1.1.0
|
||||
property-information: 6.5.0
|
||||
space-separated-tokens: 2.0.2
|
||||
web-namespaces: 2.0.1
|
||||
zwitch: 2.0.4
|
||||
|
||||
hast-util-to-text@4.0.2:
|
||||
dependencies:
|
||||
'@types/hast': 3.0.4
|
||||
@@ -16175,6 +16219,8 @@ snapshots:
|
||||
|
||||
html-url-attributes@3.0.1: {}
|
||||
|
||||
html-void-elements@3.0.0: {}
|
||||
|
||||
htmlparser2@8.0.2:
|
||||
dependencies:
|
||||
domelementtype: 2.3.0
|
||||
@@ -18238,7 +18284,7 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
next-i18next@15.4.2(i18next@23.16.8)(next@14.2.28(@babel/core@7.26.10)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
|
||||
next-i18next@15.4.2(i18next@23.16.8)(next@14.2.28(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(sass@1.85.1))(react-i18next@14.1.2(i18next@23.16.8)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1):
|
||||
dependencies:
|
||||
'@babel/runtime': 7.26.10
|
||||
'@types/hoist-non-react-statics': 3.3.6
|
||||
@@ -18873,6 +18919,8 @@ snapshots:
|
||||
dependencies:
|
||||
xtend: 4.0.2
|
||||
|
||||
property-information@6.5.0: {}
|
||||
|
||||
property-information@7.0.0: {}
|
||||
|
||||
proto-list@1.2.4: {}
|
||||
@@ -19292,6 +19340,12 @@ snapshots:
|
||||
unist-util-visit-parents: 6.0.1
|
||||
vfile: 6.0.3
|
||||
|
||||
rehype-raw@7.0.0:
|
||||
dependencies:
|
||||
'@types/hast': 3.0.4
|
||||
hast-util-raw: 9.1.0
|
||||
vfile: 6.0.3
|
||||
|
||||
remark-breaks@4.0.0:
|
||||
dependencies:
|
||||
'@types/mdast': 4.0.4
|
||||
|
||||
@@ -11,6 +11,53 @@ const nextConfig = {
|
||||
output: 'standalone',
|
||||
reactStrictMode: isDev ? false : true,
|
||||
compress: true,
|
||||
|
||||
headers: async () => {
|
||||
const nonce = Buffer.from(crypto.randomUUID()).toString('base64');
|
||||
const csp = `'nonce-${nonce}'`;
|
||||
const scheme_source = 'data: mediastream: blob: filesystem:';
|
||||
const NECESSARY_DOMAINS = [
|
||||
'*.sentry.io',
|
||||
'http://localhost:*',
|
||||
'http://127.0.0.1:*',
|
||||
'https://analytics.google.com',
|
||||
'googletagmanager.com',
|
||||
'*.googletagmanager.com',
|
||||
'https://www.google-analytics.com',
|
||||
'https://api.github.com'
|
||||
].join(' ');
|
||||
|
||||
return [
|
||||
{
|
||||
source: '/chat/(.*)',
|
||||
headers: [
|
||||
{ key: 'X-Frame-Options', value: 'DENY' },
|
||||
{ key: 'X-Content-Type-Options', value: 'nosniff' },
|
||||
{ key: 'X-XSS-Protection', value: '1; mode=block' },
|
||||
{ key: 'Referrer-Policy', value: 'no-referrer' },
|
||||
{
|
||||
key: 'Content-Security-Policy',
|
||||
value: [
|
||||
`default-src 'self' ${scheme_source} ${NECESSARY_DOMAINS} ${csp}`,
|
||||
`script-src 'self' 'unsafe-inline' 'unsafe-eval' ${csp} ${NECESSARY_DOMAINS}`,
|
||||
`style-src 'self' 'unsafe-inline' ${csp} ${NECESSARY_DOMAINS}`,
|
||||
`media-src 'self' http: ${scheme_source} ${NECESSARY_DOMAINS} ${csp}`,
|
||||
`worker-src 'self' ${csp} ${NECESSARY_DOMAINS} ${scheme_source}`,
|
||||
`img-src * data: blob:`,
|
||||
`font-src 'self'`,
|
||||
`connect-src 'self' wss: https: ${scheme_source} ${NECESSARY_DOMAINS} ${csp}`,
|
||||
"object-src 'none'",
|
||||
"form-action 'self'",
|
||||
"base-uri 'self'",
|
||||
"frame-src 'self' 'allow-scripts'",
|
||||
'sandbox allow-scripts allow-same-origin allow-popups allow-forms',
|
||||
'upgrade-insecure-requests'
|
||||
].join('; ')
|
||||
}
|
||||
]
|
||||
}
|
||||
];
|
||||
},
|
||||
webpack(config, { isServer, nextRuntime }) {
|
||||
Object.assign(config.resolve.alias, {
|
||||
'@mongodb-js/zstd': false,
|
||||
@@ -85,7 +132,7 @@ const nextConfig = {
|
||||
'pg',
|
||||
'bullmq',
|
||||
'@zilliz/milvus2-sdk-node',
|
||||
"tiktoken",
|
||||
'tiktoken'
|
||||
],
|
||||
outputFileTracingRoot: path.join(__dirname, '../../'),
|
||||
instrumentationHook: true
|
||||
|
||||
@@ -60,6 +60,7 @@
|
||||
"recharts": "^2.15.0",
|
||||
"rehype-external-links": "^3.0.0",
|
||||
"rehype-katex": "^7.0.0",
|
||||
"rehype-raw": "^7.0.0",
|
||||
"remark-breaks": "^4.0.0",
|
||||
"remark-gfm": "^4.0.0",
|
||||
"remark-math": "^6.0.0",
|
||||
|
||||
35
projects/app/src/components/Markdown/errorBoundry.tsx
Normal file
35
projects/app/src/components/Markdown/errorBoundry.tsx
Normal file
@@ -0,0 +1,35 @@
|
||||
import React from 'react';
|
||||
|
||||
interface ErrorBoundaryProps {
|
||||
children: React.ReactNode;
|
||||
fallback?: React.ReactNode;
|
||||
}
|
||||
|
||||
interface ErrorBoundaryState {
|
||||
hasError: boolean;
|
||||
error: Error | null;
|
||||
}
|
||||
|
||||
class ErrorBoundary extends React.Component<ErrorBoundaryProps, ErrorBoundaryState> {
|
||||
constructor(props: ErrorBoundaryProps) {
|
||||
super(props);
|
||||
this.state = { hasError: false, error: null };
|
||||
}
|
||||
|
||||
static getDerivedStateFromError(error: Error) {
|
||||
return { hasError: true, error };
|
||||
}
|
||||
|
||||
componentDidCatch(error: Error, errorInfo: React.ErrorInfo) {
|
||||
console.error('ErrorBoundary caught an error:', error, errorInfo);
|
||||
}
|
||||
|
||||
render() {
|
||||
if (this.state.hasError) {
|
||||
return this.props.fallback || <div>Something went wrong while rendering Markdown.</div>;
|
||||
}
|
||||
return this.props.children;
|
||||
}
|
||||
}
|
||||
|
||||
export default ErrorBoundary;
|
||||
@@ -1,4 +1,4 @@
|
||||
import React, { useCallback, useMemo } from 'react';
|
||||
import React, { memo, useCallback, useMemo } from 'react';
|
||||
import ReactMarkdown from 'react-markdown';
|
||||
import 'katex/dist/katex.min.css';
|
||||
import RemarkMath from 'remark-math'; // Math syntax
|
||||
@@ -6,15 +6,15 @@ import RemarkBreaks from 'remark-breaks'; // Line break
|
||||
import RehypeKatex from 'rehype-katex'; // Math render
|
||||
import RemarkGfm from 'remark-gfm'; // Special markdown syntax
|
||||
import RehypeExternalLinks from 'rehype-external-links';
|
||||
|
||||
import RehypeRaw from 'rehype-raw';
|
||||
import styles from './index.module.scss';
|
||||
import dynamic from 'next/dynamic';
|
||||
|
||||
import { Box } from '@chakra-ui/react';
|
||||
import { CodeClassNameEnum, mdTextFormat } from './utils';
|
||||
import ErrorBoundary from './errorBoundry';
|
||||
import SVGRenderer from './markdowSVG';
|
||||
import { useCreation } from 'ahooks';
|
||||
import type { AProps } from './A';
|
||||
|
||||
const CodeLight = dynamic(() => import('./codeBlock/CodeLight'), { ssr: false });
|
||||
const MermaidCodeBlock = dynamic(() => import('./img/MermaidCodeBlock'), { ssr: false });
|
||||
const MdImage = dynamic(() => import('./img/Image'), { ssr: false });
|
||||
@@ -26,7 +26,40 @@ const AudioBlock = dynamic(() => import('./codeBlock/Audio'), { ssr: false });
|
||||
|
||||
const ChatGuide = dynamic(() => import('./chat/Guide'), { ssr: false });
|
||||
const QuestionGuide = dynamic(() => import('./chat/QuestionGuide'), { ssr: false });
|
||||
const A = dynamic(() => import('./A'), { ssr: false });
|
||||
|
||||
function isSafeHref(href: string): boolean {
|
||||
if (!href) return false;
|
||||
// allow http(s), mailto, tel, relative paths, #, data:image/audio/video
|
||||
return /^(https?:|mailto:|tel:|\/|#|data:(?:image|audio|video))/i.test(href.trim());
|
||||
}
|
||||
|
||||
const SafeA = (props: any) => {
|
||||
const href = props.href || '';
|
||||
const safeHref = isSafeHref(href) ? href : '#';
|
||||
|
||||
const ALLOWED_A_ATTRS = new Set([
|
||||
'href',
|
||||
'target',
|
||||
'rel',
|
||||
'className',
|
||||
'children',
|
||||
'style',
|
||||
'title'
|
||||
]);
|
||||
const safeProps = filterSafeProps(props, ALLOWED_A_ATTRS);
|
||||
|
||||
return (
|
||||
<a
|
||||
{...safeProps}
|
||||
href={safeHref}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="cursor-pointer underline !decoration-primary-700 decoration-dashed"
|
||||
>
|
||||
{props.children || 'Download'}
|
||||
</a>
|
||||
);
|
||||
};
|
||||
|
||||
type Props = {
|
||||
source?: string;
|
||||
@@ -36,11 +69,9 @@ type Props = {
|
||||
} & AProps;
|
||||
const Markdown = (props: Props) => {
|
||||
const source = props.source || '';
|
||||
|
||||
if (source.length < 200000) {
|
||||
return <MarkdownRender {...props} />;
|
||||
}
|
||||
|
||||
return <Box whiteSpace={'pre-wrap'}>{source}</Box>;
|
||||
};
|
||||
const MarkdownRender = ({
|
||||
@@ -48,7 +79,6 @@ const MarkdownRender = ({
|
||||
showAnimation,
|
||||
isDisabled,
|
||||
forbidZhFormat,
|
||||
|
||||
chatAuthData,
|
||||
onOpenCiteModal
|
||||
}: Props) => {
|
||||
@@ -57,54 +87,116 @@ const MarkdownRender = ({
|
||||
img: Image,
|
||||
pre: RewritePre,
|
||||
code: Code,
|
||||
svg: SVGRenderer,
|
||||
script: ScriptBlock,
|
||||
a: (props: any) => (
|
||||
<A
|
||||
<SafeA
|
||||
{...props}
|
||||
showAnimation={showAnimation}
|
||||
chatAuthData={chatAuthData}
|
||||
onOpenCiteModal={onOpenCiteModal}
|
||||
showAnimation={showAnimation}
|
||||
/>
|
||||
)
|
||||
};
|
||||
}, [chatAuthData, onOpenCiteModal, showAnimation]);
|
||||
|
||||
const formatSource = useMemo(() => {
|
||||
if (showAnimation || forbidZhFormat) return source;
|
||||
return mdTextFormat(source);
|
||||
}, [forbidZhFormat, showAnimation, source]);
|
||||
|
||||
const urlTransform = useCallback((val: string) => {
|
||||
return val;
|
||||
}, []);
|
||||
const urlTransform = useCallback((val: string) => val, []);
|
||||
|
||||
return (
|
||||
<Box position={'relative'}>
|
||||
<ReactMarkdown
|
||||
className={`markdown ${styles.markdown}
|
||||
${showAnimation ? `${formatSource ? styles.waitingAnimation : styles.animation}` : ''}
|
||||
`}
|
||||
remarkPlugins={[RemarkMath, [RemarkGfm, { singleTilde: false }], RemarkBreaks]}
|
||||
rehypePlugins={[RehypeKatex, [RehypeExternalLinks, { target: '_blank' }]]}
|
||||
components={components}
|
||||
urlTransform={urlTransform}
|
||||
>
|
||||
{formatSource}
|
||||
</ReactMarkdown>
|
||||
<ErrorBoundary>
|
||||
<ReactMarkdown
|
||||
className={`markdown ${styles.markdown} ${
|
||||
showAnimation ? `${formatSource ? styles.waitingAnimation : styles.animation}` : ''
|
||||
}`}
|
||||
remarkPlugins={[RemarkMath, [RemarkGfm, { singleTilde: false }], RemarkBreaks]}
|
||||
rehypePlugins={[
|
||||
RehypeKatex,
|
||||
[RehypeExternalLinks, { target: '_blank', rel: ['noopener', 'noreferrer'] }],
|
||||
RehypeRaw as any,
|
||||
() => {
|
||||
return (tree) => {
|
||||
const iterate = (node: any) => {
|
||||
if (node.type === 'element') {
|
||||
// delete ref
|
||||
if (node.properties?.ref) delete node.properties.ref;
|
||||
|
||||
// handle invalid tag name
|
||||
if (!/^[a-z][a-z0-9]*$/i.test(node.tagName)) {
|
||||
node.type = 'text';
|
||||
node.value = `<${node.tagName}`;
|
||||
}
|
||||
|
||||
// handle properties, filter events
|
||||
if (node.properties) {
|
||||
Object.keys(node.properties).forEach((key) => {
|
||||
const keyLower = key.toLowerCase();
|
||||
// if event property (on开头)
|
||||
if (keyLower.startsWith('on')) {
|
||||
const value = node.properties[key];
|
||||
// if event value is not a function or contains suspicious content, delete the event
|
||||
if (
|
||||
typeof value === 'string' || // delete event handler in string format
|
||||
value === null ||
|
||||
value === undefined ||
|
||||
(typeof value === 'string' &&
|
||||
(value.includes('javascript:') ||
|
||||
value.includes('alert') ||
|
||||
value.includes('eval') ||
|
||||
value.includes('Function') ||
|
||||
/[\(\)\[\]\{\}]/.test(value))) // flag for executable code containing parentheses, etc.
|
||||
) {
|
||||
delete node.properties[key];
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// recursive handle child nodes
|
||||
if (node.children) node.children.forEach(iterate);
|
||||
};
|
||||
tree.children.forEach(iterate);
|
||||
};
|
||||
}
|
||||
]}
|
||||
disallowedElements={[
|
||||
'iframe',
|
||||
'head',
|
||||
'html',
|
||||
'meta',
|
||||
'link',
|
||||
'style',
|
||||
'body',
|
||||
'embed',
|
||||
'object',
|
||||
'param',
|
||||
'applet',
|
||||
'area',
|
||||
'map',
|
||||
'isindex'
|
||||
]}
|
||||
components={components}
|
||||
urlTransform={urlTransform}
|
||||
>
|
||||
{formatSource}
|
||||
</ReactMarkdown>
|
||||
</ErrorBoundary>
|
||||
{isDisabled && <Box position={'absolute'} top={0} right={0} left={0} bottom={0} />}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(Markdown);
|
||||
|
||||
/* Custom dom */
|
||||
function Code(e: any) {
|
||||
const { className, codeBlock, children } = e;
|
||||
const match = /language-(\w+)/.exec(className || '');
|
||||
const codeType = match?.[1]?.toLowerCase();
|
||||
|
||||
const strChildren = String(children);
|
||||
|
||||
const Component = useMemo(() => {
|
||||
if (codeType === CodeClassNameEnum.mermaid) {
|
||||
return <MermaidCodeBlock code={strChildren} />;
|
||||
@@ -134,22 +226,61 @@ function Code(e: any) {
|
||||
if (codeType === CodeClassNameEnum.audio) {
|
||||
return <AudioBlock code={strChildren} />;
|
||||
}
|
||||
|
||||
return (
|
||||
<CodeLight className={className} codeBlock={codeBlock} match={match}>
|
||||
{children}
|
||||
</CodeLight>
|
||||
);
|
||||
}, [codeType, className, codeBlock, match, children, strChildren]);
|
||||
|
||||
return Component;
|
||||
}
|
||||
|
||||
function Image({ src }: { src?: string }) {
|
||||
return <MdImage src={src} />;
|
||||
function sanitizeImageSrc(src?: string): string | undefined {
|
||||
if (!src) return undefined;
|
||||
// remove leading and trailing spaces
|
||||
const trimmed = src.trim();
|
||||
// only allow http/https/data/blob protocols
|
||||
if (/^(https?:|data:|blob:)/i.test(trimmed)) {
|
||||
return trimmed;
|
||||
}
|
||||
// allow relative paths (not starting with javascript: or vbscript:)
|
||||
if (
|
||||
!/^(\w+:)/.test(trimmed) &&
|
||||
!trimmed.startsWith('javascript:') &&
|
||||
!trimmed.startsWith('vbscript:')
|
||||
) {
|
||||
return trimmed;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function RewritePre({ children }: any) {
|
||||
const ALLOWED_IMG_ATTRS = new Set([
|
||||
'alt',
|
||||
'width',
|
||||
'height',
|
||||
'className',
|
||||
'style',
|
||||
'title',
|
||||
'loading',
|
||||
'decoding',
|
||||
'crossOrigin',
|
||||
'referrerPolicy'
|
||||
]);
|
||||
|
||||
function Image({ src, ...rest }: { src?: string; [key: string]: any }) {
|
||||
const safeSrc = sanitizeImageSrc(src);
|
||||
if (!safeSrc) {
|
||||
console.warn(`Blocked unsafe image src: ${src}`);
|
||||
}
|
||||
// only allow whitelist attributes, and remove all on* events
|
||||
const safeProps = filterSafeProps(rest, ALLOWED_IMG_ATTRS);
|
||||
return <MdImage src={safeSrc} {...safeProps} />;
|
||||
}
|
||||
|
||||
function RewritePre({ children, ...rest }: any) {
|
||||
// only allow className, style, etc. safe attributes
|
||||
const ALLOWED_PRE_ATTRS = new Set(['className', 'style']);
|
||||
const safeProps = filterSafeProps(rest, ALLOWED_PRE_ATTRS);
|
||||
const modifiedChildren = React.Children.map(children, (child) => {
|
||||
if (React.isValidElement(child)) {
|
||||
// @ts-ignore
|
||||
@@ -157,6 +288,146 @@ function RewritePre({ children }: any) {
|
||||
}
|
||||
return child;
|
||||
});
|
||||
|
||||
return <>{modifiedChildren}</>;
|
||||
return <pre {...safeProps}>{modifiedChildren}</pre>;
|
||||
}
|
||||
|
||||
/**
|
||||
* general safe attribute filter
|
||||
* @param props input props object
|
||||
* @param allowedAttrs allowed attribute name Set
|
||||
* @param eventTypeCheck whether to check event type (e.g. onClick must be a function)
|
||||
*/
|
||||
export function filterSafeProps(
|
||||
props: Record<string, any>,
|
||||
allowedAttrs: Set<string>,
|
||||
eventTypeCheck: boolean = true
|
||||
) {
|
||||
// dangerous protocols
|
||||
const DANGEROUS_PROTOCOLS =
|
||||
/^(?:\s| | )*(?:javascript|vbscript|data(?!:(?:image|audio|video)))/i;
|
||||
|
||||
// dangerous event properties (including various possible ways)
|
||||
const DANGEROUS_EVENTS =
|
||||
/^(?:\s| | )*(?:on|formaction|data-|\[\[|\{\{|xlink:|href|src|action)/i;
|
||||
|
||||
// complete decode function
|
||||
function fullDecode(input: string): string {
|
||||
if (!input) return '';
|
||||
|
||||
let result = input;
|
||||
let lastResult = '';
|
||||
|
||||
// continue decoding until no more decoding can be done
|
||||
while (result !== lastResult) {
|
||||
lastResult = result;
|
||||
try {
|
||||
// HTML entity decode
|
||||
result = result.replace(/&(#?[\w\d]+);/g, (_, entity) => {
|
||||
try {
|
||||
const txt = document.createElement('textarea');
|
||||
txt.innerHTML = `&${entity};`;
|
||||
return txt.value;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
// Unicode decode (\u0061 format)
|
||||
result = result.replace(/(?:\\|%5C|%5c)u([0-9a-f]{4})/gi, (_, hex) =>
|
||||
String.fromCharCode(parseInt(hex, 16))
|
||||
);
|
||||
|
||||
// URL encode decode
|
||||
result = result.replace(/%([0-9a-f]{2})/gi, (_, hex) =>
|
||||
String.fromCharCode(parseInt(hex, 16))
|
||||
);
|
||||
|
||||
// octal decode
|
||||
result = result.replace(/\\([0-7]{3})/gi, (_, oct) =>
|
||||
String.fromCharCode(parseInt(oct, 8))
|
||||
);
|
||||
|
||||
// hexadecimal decode (\x61 format)
|
||||
result = result.replace(/(?:\\|%5C|%5c)x([0-9a-f]{2})/gi, (_, hex) =>
|
||||
String.fromCharCode(parseInt(hex, 16))
|
||||
);
|
||||
|
||||
// handle whitespace and comments
|
||||
result = result.replace(/(?:\s|\/\*.*?\*\/|<!--.*?-->)+/g, '');
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return result.toLowerCase();
|
||||
}
|
||||
|
||||
// check if it contains dangerous content
|
||||
function containsDangerousContent(value: string): boolean {
|
||||
const decoded = fullDecode(value);
|
||||
|
||||
return (
|
||||
// check dangerous protocol
|
||||
DANGEROUS_PROTOCOLS.test(decoded) ||
|
||||
// check dangerous event
|
||||
DANGEROUS_EVENTS.test(decoded) ||
|
||||
// check inline event
|
||||
/on\w+\s*=/.test(decoded) ||
|
||||
// check javascript: link
|
||||
/javascript\s*:/.test(decoded) ||
|
||||
// check other possible injections
|
||||
/<\w+/i.test(decoded) ||
|
||||
/\(\s*\)/i.test(decoded) ||
|
||||
/\[\s*\]/i.test(decoded) ||
|
||||
/\{\s*\}/i.test(decoded)
|
||||
);
|
||||
}
|
||||
|
||||
return Object.fromEntries(
|
||||
Object.entries(props).filter(([key, value]) => {
|
||||
// 1. decode and check property name
|
||||
const decodedKey = fullDecode(key);
|
||||
|
||||
// 2. intercept all event related properties
|
||||
if (DANGEROUS_EVENTS.test(decodedKey)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 3. all properties not in the whitelist are rejected
|
||||
if (!allowedAttrs.has(key)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 4. check property value
|
||||
if (typeof value === 'string') {
|
||||
if (containsDangerousContent(value)) {
|
||||
return false;
|
||||
}
|
||||
} else if (typeof value === 'object' && value !== null) {
|
||||
// only allow simple style objects
|
||||
if (key !== 'style') {
|
||||
return false;
|
||||
}
|
||||
// check the value of the style object
|
||||
for (const styleKey in value) {
|
||||
if (containsDangerousContent(String(value[styleKey]))) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
} else if (typeof value === 'function') {
|
||||
// only allow specific function properties (e.g. onClick)
|
||||
if (!eventTypeCheck || decodedKey !== 'onclick') {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
const ScriptBlock = memo(({ node }: any) => {
|
||||
const scriptContent = node.children[0]?.value || '';
|
||||
return `<script>${scriptContent}</script>`;
|
||||
});
|
||||
ScriptBlock.displayName = 'ScriptBlock';
|
||||
|
||||
108
projects/app/src/components/Markdown/markdowSVG.tsx
Normal file
108
projects/app/src/components/Markdown/markdowSVG.tsx
Normal file
@@ -0,0 +1,108 @@
|
||||
import React from 'react';
|
||||
import ErrorBoundary from './errorBoundry';
|
||||
import { filterSafeProps } from './index';
|
||||
|
||||
interface SVGProps {
|
||||
children?: React.ReactNode;
|
||||
className?: string;
|
||||
style?: React.CSSProperties;
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
const SVG_ALLOWED_ATTRS = new Set([
|
||||
'width',
|
||||
'height',
|
||||
'viewBox',
|
||||
'fill',
|
||||
'stroke',
|
||||
'd',
|
||||
'x',
|
||||
'y',
|
||||
'cx',
|
||||
'cy',
|
||||
'r',
|
||||
'className',
|
||||
'style'
|
||||
]);
|
||||
|
||||
const SVGRenderer = ({ children, className, style, ...props }: SVGProps) => {
|
||||
// filter props
|
||||
const svgProps = { ...props, className, style };
|
||||
const sanitizedProps = filterSafeProps(svgProps, SVG_ALLOWED_ATTRS, false);
|
||||
|
||||
const sanitizeSVGContent = (content: string | React.ReactNode): string => {
|
||||
if (typeof content !== 'string') {
|
||||
return '';
|
||||
}
|
||||
|
||||
let cleaned = content;
|
||||
|
||||
cleaned = cleaned.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
|
||||
cleaned = cleaned.replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, '');
|
||||
cleaned = cleaned.replace(
|
||||
/<foreignObject\b[^<]*(?:(?!<\/foreignObject>)<[^<]*)*<\/foreignObject>/gi,
|
||||
''
|
||||
);
|
||||
|
||||
cleaned = cleaned.replace(/\son\w+="[^"]*"/gi, '');
|
||||
cleaned = cleaned.replace(/\son\w+='[^']*'/gi, '');
|
||||
cleaned = cleaned.replace(/url\s*\(\s*['"]?\s*javascript:[^)]+\)/gi, '');
|
||||
cleaned = cleaned.replace(/\bhref="javascript:[^"]*"/gi, '');
|
||||
cleaned = cleaned.replace(/\bhref='javascript:[^']*'/gi, '');
|
||||
cleaned = cleaned.replace(/\bxlink:href="javascript:[^"]*"/gi, '');
|
||||
cleaned = cleaned.replace(/\bxlink:href='javascript:[^']*'/gi, '');
|
||||
cleaned = cleaned.replace(/\bxmlns(:xlink)?=['"]?javascript:[^"']*['"]?/gi, '');
|
||||
cleaned = cleaned.replace(/style\s*=\s*(['"])(?:(?!\1).)*javascript:.*?\1/gi, '');
|
||||
|
||||
cleaned = cleaned.replace(/\bdata:[^,]*?;base64,[^"')]*["')]/gi, (match) => {
|
||||
return match.toLowerCase().includes('javascript') ? '' : match;
|
||||
});
|
||||
|
||||
const ALLOWED_ATTRS = new Set([
|
||||
'width',
|
||||
'height',
|
||||
'viewBox',
|
||||
'fill',
|
||||
'stroke',
|
||||
'd',
|
||||
'x',
|
||||
'y',
|
||||
'cx',
|
||||
'cy',
|
||||
'r',
|
||||
'class',
|
||||
'style'
|
||||
]);
|
||||
cleaned = cleaned.replace(/\s(\w+)=['"][^'"]*['"]/gi, (match, attr) => {
|
||||
return ALLOWED_ATTRS.has(attr.toLowerCase()) ? match : '';
|
||||
});
|
||||
|
||||
cleaned = cleaned.replace(/<!--[\s\S]*?-->/g, '');
|
||||
|
||||
return cleaned;
|
||||
};
|
||||
|
||||
const sanitizedContent = React.Children.map(children, (child) => {
|
||||
if (typeof child === 'string') {
|
||||
return sanitizeSVGContent(child);
|
||||
}
|
||||
return child;
|
||||
});
|
||||
|
||||
return (
|
||||
<ErrorBoundary fallback={<div>Something went wrong while rendering Markdown.</div>}>
|
||||
<svg
|
||||
{...sanitizedProps}
|
||||
className={className}
|
||||
style={style}
|
||||
dangerouslySetInnerHTML={
|
||||
typeof children === 'string' ? { __html: sanitizeSVGContent(children) } : undefined
|
||||
}
|
||||
>
|
||||
{typeof children !== 'string' && sanitizedContent}
|
||||
</svg>
|
||||
</ErrorBoundary>
|
||||
);
|
||||
};
|
||||
|
||||
export default SVGRenderer;
|
||||
@@ -39,6 +39,12 @@ export async function register() {
|
||||
systemStartCb();
|
||||
initGlobalVariables();
|
||||
|
||||
try {
|
||||
await preLoadWorker();
|
||||
} catch (error) {
|
||||
console.error('Preload worker error', error);
|
||||
}
|
||||
|
||||
// Connect to MongoDB
|
||||
await connectMongo(connectionMongo, MONGO_URL);
|
||||
connectMongo(connectionLogMongo, MONGO_LOG_URL);
|
||||
@@ -54,12 +60,6 @@ export async function register() {
|
||||
startCron();
|
||||
startTrainingQueue(true);
|
||||
|
||||
try {
|
||||
await preLoadWorker();
|
||||
} catch (error) {
|
||||
console.error('Preload worker error', error);
|
||||
}
|
||||
|
||||
console.log('Init system success');
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -25,16 +25,20 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
|
||||
import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
|
||||
import { useToast } from '@fastgpt/web/hooks/useToast';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import SaveButton from '../Workflow/components/SaveButton';
|
||||
import PublishHistories from '../PublishHistoriesSlider';
|
||||
import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
|
||||
import { WorkflowStatusContext } from '../WorkflowComponents/context/workflowStatusContext';
|
||||
import SaveButton from '../Workflow/components/SaveButton';
|
||||
|
||||
const Header = () => {
|
||||
const { t } = useTranslation();
|
||||
const { isPc } = useSystem();
|
||||
const router = useRouter();
|
||||
const { toast } = useToast();
|
||||
const { toast: backSaveToast } = useToast({
|
||||
containerStyle: {
|
||||
mt: '60px'
|
||||
}
|
||||
});
|
||||
|
||||
const { appDetail, onSaveApp, currentTab } = useContextSelector(AppContext, (v) => v);
|
||||
const isV2Workflow = appDetail?.version === 'v2';
|
||||
@@ -183,6 +187,7 @@ const Header = () => {
|
||||
size={'sm'}
|
||||
leftIcon={<MyIcon name={'core/workflow/debug'} w={['14px', '16px']} />}
|
||||
variant={'whitePrimary'}
|
||||
flexShrink={0}
|
||||
onClick={() => {
|
||||
const data = flowData2StoreDataAndCheck();
|
||||
if (data) {
|
||||
@@ -211,12 +216,12 @@ const Header = () => {
|
||||
onBack,
|
||||
onOpenBackConfirm,
|
||||
isV2Workflow,
|
||||
showHistoryModal,
|
||||
t,
|
||||
showHistoryModal,
|
||||
loading,
|
||||
onClickSave,
|
||||
flowData2StoreDataAndCheck,
|
||||
setShowHistoryModal,
|
||||
flowData2StoreDataAndCheck,
|
||||
setWorkflowTestData
|
||||
]);
|
||||
|
||||
@@ -229,10 +234,11 @@ const Header = () => {
|
||||
setShowHistoryModal(false);
|
||||
}}
|
||||
past={past}
|
||||
onSwitchTmpVersion={onSwitchTmpVersion}
|
||||
onSwitchCloudVersion={onSwitchCloudVersion}
|
||||
onSwitchTmpVersion={onSwitchTmpVersion}
|
||||
/>
|
||||
)}
|
||||
|
||||
<MyModal
|
||||
isOpen={isOpenBackConfirm}
|
||||
onClose={onCloseBackConfirm}
|
||||
@@ -254,7 +260,7 @@ const Header = () => {
|
||||
await onClickSave({});
|
||||
onCloseBackConfirm();
|
||||
onBack();
|
||||
toast({
|
||||
backSaveToast({
|
||||
status: 'success',
|
||||
title: t('app:saved_success'),
|
||||
position: 'top-right'
|
||||
|
||||
@@ -13,7 +13,7 @@ import { useTranslation } from 'next-i18next';
|
||||
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { WorkflowContext } from '../WorkflowComponents/context';
|
||||
import { WorkflowContext, type WorkflowSnapshotsType } from '../WorkflowComponents/context';
|
||||
import { AppContext, TabEnum } from '../context';
|
||||
import RouteTab from '../RouteTab';
|
||||
import { useRouter } from 'next/router';
|
||||
@@ -25,10 +25,10 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
|
||||
import { formatTime2YMDHMS } from '@fastgpt/global/common/string/time';
|
||||
import { useToast } from '@fastgpt/web/hooks/useToast';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import SaveButton from './components/SaveButton';
|
||||
import PublishHistories from '../PublishHistoriesSlider';
|
||||
import { WorkflowEventContext } from '../WorkflowComponents/context/workflowEventContext';
|
||||
import { WorkflowStatusContext } from '../WorkflowComponents/context/workflowStatusContext';
|
||||
import SaveButton from '../Workflow/components/SaveButton';
|
||||
|
||||
const Header = () => {
|
||||
const { t } = useTranslation();
|
||||
@@ -187,6 +187,7 @@ const Header = () => {
|
||||
size={'sm'}
|
||||
leftIcon={<MyIcon name={'core/workflow/debug'} w={['14px', '16px']} />}
|
||||
variant={'whitePrimary'}
|
||||
flexShrink={0}
|
||||
onClick={() => {
|
||||
const data = flowData2StoreDataAndCheck();
|
||||
if (data) {
|
||||
@@ -215,12 +216,12 @@ const Header = () => {
|
||||
onBack,
|
||||
onOpenBackConfirm,
|
||||
isV2Workflow,
|
||||
showHistoryModal,
|
||||
t,
|
||||
showHistoryModal,
|
||||
loading,
|
||||
onClickSave,
|
||||
flowData2StoreDataAndCheck,
|
||||
setShowHistoryModal,
|
||||
flowData2StoreDataAndCheck,
|
||||
setWorkflowTestData
|
||||
]);
|
||||
|
||||
@@ -228,7 +229,7 @@ const Header = () => {
|
||||
<>
|
||||
{Render}
|
||||
{showHistoryModal && isV2Workflow && currentTab === TabEnum.appEdit && (
|
||||
<PublishHistories
|
||||
<PublishHistories<WorkflowSnapshotsType>
|
||||
onClose={() => {
|
||||
setShowHistoryModal(false);
|
||||
}}
|
||||
|
||||
@@ -43,6 +43,7 @@ const SaveButton = ({
|
||||
Trigger={
|
||||
<Button
|
||||
size={'sm'}
|
||||
flexShrink={0}
|
||||
rightIcon={
|
||||
<MyIcon
|
||||
name={isSave ? 'core/chat/chevronUp' : 'core/chat/chevronDown'}
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
import React, { useState, useCallback } from 'react';
|
||||
import { Box, Flex, Button, IconButton, type ButtonProps, Input } from '@chakra-ui/react';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { WorkflowNodeEdgeContext } from '../../WorkflowComponents/context/workflowInitContext';
|
||||
import { useReactFlow } from 'reactflow';
|
||||
import { useKeyPress, useThrottleEffect } from 'ahooks';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
|
||||
import { useSystem } from '@fastgpt/web/hooks/useSystem';
|
||||
|
||||
const SearchButton = (props: ButtonProps) => {
|
||||
const { t } = useTranslation();
|
||||
const setNodes = useContextSelector(WorkflowNodeEdgeContext, (state) => state.setNodes);
|
||||
const { fitView } = useReactFlow();
|
||||
const { isMac } = useSystem();
|
||||
|
||||
const [keyword, setKeyword] = useState<string>();
|
||||
const [searchIndex, setSearchIndex] = useState<number>(0);
|
||||
const [searchedNodeCount, setSearchedNodeCount] = useState(0);
|
||||
|
||||
useKeyPress(['ctrl.f', 'meta.f'], (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
setKeyword('');
|
||||
});
|
||||
useKeyPress(['esc'], (e) => {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
setKeyword(undefined);
|
||||
});
|
||||
|
||||
const onSearch = useCallback(() => {
|
||||
setNodes((nodes) => {
|
||||
if (!keyword) {
|
||||
setSearchIndex(0);
|
||||
setSearchedNodeCount(0);
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
data: {
|
||||
...node.data,
|
||||
searchedText: undefined
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
const searchResult = nodes.filter((node) => {
|
||||
const nodeName = t(node.data.name as any);
|
||||
return nodeName.toLowerCase().includes(keyword.toLowerCase());
|
||||
});
|
||||
|
||||
if (searchResult.length === 0) {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
data: {
|
||||
...node.data,
|
||||
searchedText: undefined
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
setSearchedNodeCount(searchResult.length);
|
||||
|
||||
const searchedNode = searchResult[searchIndex] ?? searchResult[0];
|
||||
|
||||
if (searchedNode) {
|
||||
fitView({ nodes: [searchedNode], padding: 0.4 });
|
||||
}
|
||||
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
selected: node.id === searchedNode.id,
|
||||
data: {
|
||||
...node.data,
|
||||
searchedText: searchResult.find((item) => item.id === node.id) ? keyword : undefined
|
||||
}
|
||||
}));
|
||||
});
|
||||
}, [keyword, searchIndex]);
|
||||
|
||||
useThrottleEffect(
|
||||
() => {
|
||||
onSearch();
|
||||
},
|
||||
[onSearch],
|
||||
{
|
||||
wait: 500
|
||||
}
|
||||
);
|
||||
|
||||
const goToNextMatch = useCallback(() => {
|
||||
if (searchIndex === searchedNodeCount - 1) {
|
||||
setSearchIndex(0);
|
||||
} else {
|
||||
setSearchIndex(searchIndex + 1);
|
||||
}
|
||||
}, [searchIndex, searchedNodeCount]);
|
||||
|
||||
const goToPreviousMatch = useCallback(() => {
|
||||
if (searchIndex === 0) {
|
||||
setSearchIndex(searchedNodeCount - 1);
|
||||
} else {
|
||||
setSearchIndex(searchIndex - 1);
|
||||
}
|
||||
}, [searchIndex, searchedNodeCount]);
|
||||
|
||||
const clearSearch = useCallback(() => {
|
||||
setKeyword(undefined);
|
||||
setSearchIndex(0);
|
||||
setSearchedNodeCount(0);
|
||||
}, []);
|
||||
|
||||
if (keyword === undefined) {
|
||||
return (
|
||||
<Box position={'absolute'} top={'72px'} left={6} zIndex={1}>
|
||||
<MyTooltip label={isMac ? t('workflow:find_tip_mac') : t('workflow:find_tip')}>
|
||||
<IconButton
|
||||
icon={<MyIcon name="common/searchLight" w="20px" color={'#8A95A7'} />}
|
||||
aria-label=""
|
||||
variant="whitePrimary"
|
||||
size={'mdSquare'}
|
||||
borderRadius={'50%'}
|
||||
bg={'white'}
|
||||
_hover={{ bg: 'white', borderColor: 'primary.300' }}
|
||||
boxShadow={'0px 4px 10px 0px rgba(19, 51, 107, 0.20)'}
|
||||
{...props}
|
||||
onClick={() => setKeyword('')}
|
||||
/>
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<Flex
|
||||
position="absolute"
|
||||
top={3}
|
||||
left="50%"
|
||||
transform="translateX(-50%)"
|
||||
pl={5}
|
||||
pr={4}
|
||||
py={4}
|
||||
zIndex={1}
|
||||
borderRadius={'lg'}
|
||||
bg={'white'}
|
||||
alignItems={'center'}
|
||||
boxShadow={
|
||||
'0px 20px 24px -8px rgba(19, 51, 107, 0.15), 0px 0px 1px 0px rgba(19, 51, 107, 0.15)'
|
||||
}
|
||||
border={'0.5px solid rgba(0, 0, 0, 0.13)'}
|
||||
maxW={['90vw', '550px']}
|
||||
w={'100%'}
|
||||
>
|
||||
<Input
|
||||
flex="1 0 0"
|
||||
h={8}
|
||||
border={'none'}
|
||||
px={0}
|
||||
_focus={{
|
||||
border: 'none',
|
||||
boxShadow: 'none'
|
||||
}}
|
||||
fontSize={'16px'}
|
||||
value={keyword}
|
||||
placeholder={t('workflow:please_enter_node_name')}
|
||||
autoFocus
|
||||
onFocus={onSearch}
|
||||
onChange={(e) => setKeyword(e.target.value)}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
goToNextMatch();
|
||||
}
|
||||
}}
|
||||
/>
|
||||
|
||||
<Box fontSize="sm" color="myGray.600" whiteSpace={'nowrap'} userSelect={'none'}>
|
||||
{searchedNodeCount > 0
|
||||
? `${searchIndex + 1} / ${searchedNodeCount}`
|
||||
: t('workflow:no_match_node')}
|
||||
</Box>
|
||||
|
||||
{/* Border */}
|
||||
<Box h={5} w={'1px'} bg={'myGray.250'} ml={3} mr={2} />
|
||||
|
||||
<Button
|
||||
size="xs"
|
||||
variant="grayGhost"
|
||||
px={2}
|
||||
isDisabled={searchedNodeCount <= 1}
|
||||
onClick={goToPreviousMatch}
|
||||
>
|
||||
{t('workflow:previous')}
|
||||
</Button>
|
||||
<Button
|
||||
size="xs"
|
||||
variant="grayGhost"
|
||||
px={2}
|
||||
isDisabled={searchedNodeCount <= 1}
|
||||
onClick={goToNextMatch}
|
||||
>
|
||||
{t('workflow:next')}
|
||||
</Button>
|
||||
|
||||
<Flex
|
||||
ml={2}
|
||||
borderRadius="sm"
|
||||
_hover={{ bg: 'myGray.100' }}
|
||||
p={'1'}
|
||||
cursor="pointer"
|
||||
onClick={clearSearch}
|
||||
>
|
||||
<MyIcon name="common/closeLight" w="1.2rem" />
|
||||
</Flex>
|
||||
</Flex>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(SearchButton);
|
||||
@@ -1,7 +1,6 @@
|
||||
import React from 'react';
|
||||
import ReactFlow, { type NodeProps, SelectionMode } from 'reactflow';
|
||||
import { Box, IconButton, useDisclosure } from '@chakra-ui/react';
|
||||
import { SmallCloseIcon } from '@chakra-ui/icons';
|
||||
import { EDGE_TYPE, FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
@@ -20,6 +19,8 @@ import ContextMenu from './components/ContextMenu';
|
||||
import { WorkflowNodeEdgeContext, WorkflowInitContext } from '../context/workflowInitContext';
|
||||
import { WorkflowEventContext } from '../context/workflowEventContext';
|
||||
import NodeTemplatesPopover from './NodeTemplatesPopover';
|
||||
import SearchButton from '../../Workflow/components/SearchButton';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
|
||||
const NodeSimple = dynamic(() => import('./nodes/NodeSimple'));
|
||||
const nodeTypes: Record<FlowNodeTypeEnum, any> = {
|
||||
@@ -113,20 +114,22 @@ const Workflow = () => {
|
||||
<>
|
||||
<IconButton
|
||||
position={'absolute'}
|
||||
top={5}
|
||||
left={5}
|
||||
top={6}
|
||||
left={6}
|
||||
size={'mdSquare'}
|
||||
borderRadius={'50%'}
|
||||
icon={<SmallCloseIcon fontSize={'26px'} />}
|
||||
transform={isOpenTemplate ? '' : 'rotate(135deg)'}
|
||||
icon={<MyIcon name="common/addLight" w={'26px'} />}
|
||||
transition={'0.2s ease'}
|
||||
aria-label={''}
|
||||
zIndex={1}
|
||||
boxShadow={'2px 2px 6px #85b1ff'}
|
||||
boxShadow={
|
||||
'0px 4px 10px 0px rgba(19, 51, 107, 0.20), 0px 0px 1px 0px rgba(19, 51, 107, 0.50)'
|
||||
}
|
||||
onClick={() => {
|
||||
isOpenTemplate ? onCloseTemplate() : onOpenTemplate();
|
||||
}}
|
||||
/>
|
||||
<SearchButton />
|
||||
<NodeTemplatesModal isOpen={isOpenTemplate} onClose={onCloseTemplate} />
|
||||
<NodeTemplatesPopover />
|
||||
</>
|
||||
|
||||
@@ -36,6 +36,7 @@ import MyTag from '@fastgpt/web/components/common/Tag/index';
|
||||
import MySelect from '@fastgpt/web/components/common/MySelect';
|
||||
import { useCreation } from 'ahooks';
|
||||
import { formatToolError } from '@fastgpt/global/core/app/utils';
|
||||
import HighlightText from '@fastgpt/web/components/common/String/HighlightText';
|
||||
|
||||
type Props = FlowNodeItemType & {
|
||||
children?: React.ReactNode | React.ReactNode[] | string;
|
||||
@@ -45,6 +46,7 @@ type Props = FlowNodeItemType & {
|
||||
w?: string | number;
|
||||
h?: string | number;
|
||||
selected?: boolean;
|
||||
searchedText?: string;
|
||||
menuForbid?: {
|
||||
debug?: boolean;
|
||||
copy?: boolean;
|
||||
@@ -70,6 +72,7 @@ const NodeCard = (props: Props) => {
|
||||
h = 'full',
|
||||
nodeId,
|
||||
selected,
|
||||
searchedText,
|
||||
menuForbid,
|
||||
isTool = false,
|
||||
isError = false,
|
||||
@@ -187,7 +190,12 @@ const NodeCard = (props: Props) => {
|
||||
h={'24px'}
|
||||
/>
|
||||
<Box ml={2} fontSize={'18px'} fontWeight={'medium'} color={'myGray.900'}>
|
||||
{t(name as any)}
|
||||
<HighlightText
|
||||
rawText={t(name as any)}
|
||||
matchText={searchedText ?? ''}
|
||||
mode={'bg'}
|
||||
color={'#ffe82d'}
|
||||
/>
|
||||
</Box>
|
||||
<Button
|
||||
display={'none'}
|
||||
@@ -280,6 +288,7 @@ const NodeCard = (props: Props) => {
|
||||
nodeId,
|
||||
isFolded,
|
||||
avatar,
|
||||
searchedText,
|
||||
t,
|
||||
name,
|
||||
showVersion,
|
||||
|
||||
@@ -49,7 +49,7 @@ const CustomTextInput = () => {
|
||||
createStatus: 'waiting',
|
||||
rawText: data.value,
|
||||
sourceName: data.name,
|
||||
icon: 'file/fill/manual'
|
||||
icon: 'file/fill/txt'
|
||||
}
|
||||
]);
|
||||
goToNext();
|
||||
|
||||
@@ -138,18 +138,20 @@ async function handler(req: ApiRequestProps<ListAppBody>): Promise<AppListItemTy
|
||||
})();
|
||||
const limit = (() => {
|
||||
if (getRecentlyChat) return 15;
|
||||
if (searchKey) return 20;
|
||||
return 1000;
|
||||
if (searchKey) return 50;
|
||||
return;
|
||||
})();
|
||||
|
||||
const myApps = await MongoApp.find(
|
||||
findAppsQuery,
|
||||
'_id parentId avatar type name intro tmbId updateTime pluginData inheritPermission'
|
||||
'_id parentId avatar type name intro tmbId updateTime pluginData inheritPermission',
|
||||
{
|
||||
limit: limit
|
||||
}
|
||||
)
|
||||
.sort({
|
||||
updateTime: -1
|
||||
})
|
||||
.limit(limit)
|
||||
.lean();
|
||||
|
||||
// Add app permission and filter apps by read permission
|
||||
|
||||
@@ -4,11 +4,11 @@ import { type FileIdCreateDatasetCollectionParams } from '@fastgpt/global/core/d
|
||||
import { createCollectionAndInsertData } from '@fastgpt/service/core/dataset/collection/controller';
|
||||
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
|
||||
import { MongoRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/schema';
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { type ApiRequestProps } from '@fastgpt/service/type/next';
|
||||
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { type CreateCollectionResponse } from '@/global/core/dataset/api';
|
||||
import { deleteRawTextBuffer } from '@fastgpt/service/common/buffer/rawText/controller';
|
||||
|
||||
async function handler(
|
||||
req: ApiRequestProps<FileIdCreateDatasetCollectionParams>
|
||||
@@ -52,7 +52,7 @@ async function handler(
|
||||
});
|
||||
|
||||
// remove buffer
|
||||
await MongoRawTextBuffer.deleteOne({ sourceId: fileId });
|
||||
await deleteRawTextBuffer(fileId);
|
||||
|
||||
return {
|
||||
collectionId,
|
||||
|
||||
@@ -6,6 +6,7 @@ import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant
|
||||
import { NextAPI } from '@/service/middleware/entry';
|
||||
import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
|
||||
import { type CreateCollectionResponse } from '@/global/core/dataset/api';
|
||||
import { createFileFromText } from '@fastgpt/service/common/file/gridfs/utils';
|
||||
|
||||
async function handler(req: NextApiRequest): CreateCollectionResponse {
|
||||
const { name, text, ...body } = req.body as TextCreateDatasetCollectionParams;
|
||||
@@ -18,6 +19,18 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
|
||||
per: WritePermissionVal
|
||||
});
|
||||
|
||||
// 1. Create file from text
|
||||
const filename = `${name}.txt`;
|
||||
const { fileId } = await createFileFromText({
|
||||
bucket: 'dataset',
|
||||
filename,
|
||||
text,
|
||||
metadata: {
|
||||
teamId,
|
||||
uid: tmbId
|
||||
}
|
||||
});
|
||||
|
||||
const { collectionId, insertResults } = await createCollectionAndInsertData({
|
||||
dataset,
|
||||
rawText: text,
|
||||
@@ -25,9 +38,9 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
|
||||
...body,
|
||||
teamId,
|
||||
tmbId,
|
||||
type: DatasetCollectionTypeEnum.virtual,
|
||||
|
||||
name
|
||||
type: DatasetCollectionTypeEnum.file,
|
||||
fileId,
|
||||
name: filename
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import { checkTimerLock } from '@fastgpt/service/common/system/timerLock/utils';
|
||||
import { TimerIdEnum } from '@fastgpt/service/common/system/timerLock/constants';
|
||||
import { addHours } from 'date-fns';
|
||||
import { getScheduleTriggerApp } from '@/service/core/app/utils';
|
||||
import { clearExpiredRawTextBufferCron } from '@fastgpt/service/common/buffer/rawText/controller';
|
||||
|
||||
// Try to run train every minute
|
||||
const setTrainingQueueCron = () => {
|
||||
@@ -83,4 +84,5 @@ export const startCron = () => {
|
||||
setClearTmpUploadFilesCron();
|
||||
clearInvalidDataCron();
|
||||
scheduleTriggerAppCron();
|
||||
clearExpiredRawTextBufferCron();
|
||||
};
|
||||
|
||||
@@ -2,7 +2,7 @@ import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/sch
|
||||
import { pushQAUsage } from '@/service/support/wallet/usage/push';
|
||||
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { createChatCompletion } from '@fastgpt/service/core/ai/config';
|
||||
import type { ChatCompletionMessageParam, StreamChatType } from '@fastgpt/global/core/ai/type.d';
|
||||
import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d';
|
||||
import { addLog } from '@fastgpt/service/common/system/log';
|
||||
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
|
||||
import { replaceVariable } from '@fastgpt/global/common/string/tools';
|
||||
@@ -1,6 +1,6 @@
|
||||
import { TeamErrEnum } from '@fastgpt/global/common/error/code/team';
|
||||
import { checkTeamAIPoints } from '@fastgpt/service/support/permission/teamLimit';
|
||||
import { sendOneInform } from '../support/user/inform/api';
|
||||
import { sendOneInform } from '../../../support/user/inform/api';
|
||||
import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller';
|
||||
import { InformLevelEnum } from '@fastgpt/global/support/user/inform/constants';
|
||||
|
||||
@@ -18,7 +18,7 @@ export const checkTeamAiPointsAndLock = async (teamId: string) => {
|
||||
templateParam: {},
|
||||
teamId
|
||||
});
|
||||
console.log('余额不足,暂停【向量】生成任务');
|
||||
console.log('余额不足,暂停训练生成任务');
|
||||
await lockTrainingDataByTeamId(teamId);
|
||||
} catch (error) {}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { generateQA } from '@/service/events/generateQA';
|
||||
import { generateVector } from '@/service/events/generateVector';
|
||||
import { generateQA } from '@/service/core/dataset/queues/generateQA';
|
||||
import { generateVector } from '@/service/core/dataset/queues/generateVector';
|
||||
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { type DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
|
||||
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
|
||||
|
||||
Reference in New Issue
Block a user