gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

4.6.7-alpha commit (#743)

Browse Source 
Co-authored-by: Archer <545436317@qq.com>
Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
This commit is contained in:
Archer
2024-01-19 11:17:28 +08:00
committed by GitHub
parent 8ee7407c4c
commit c031e6dcc9
324 changed files with 8509 additions and 4757 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
projects/app/src/pages/_app.tsx
View File

@@ -1,4 +1,4 @@
import { useEffect, useRef, useState } from 'react';
import { useEffect, useState } from 'react';
import type { AppProps } from 'next/app';
import Script from 'next/script';
import Head from 'next/head';

28
projects/app/src/pages/account/components/BillDetail.tsx
View File

@@ -30,7 +30,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
hasTokens,
hasInputTokens,
hasOutputTokens,
hasTextLen,
hasCharsLen,
hasDuration,
hasDataLen
} = useMemo(() => {
@@ -38,7 +38,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
let hasTokens = false;
let hasInputTokens = false;
let hasOutputTokens = false;
let hasTextLen = false;
let hasCharsLen = false;
let hasDuration = false;
let hasDataLen = false;
@@ -46,24 +46,21 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
if (item.model !== undefined) {
hasModel = true;
}
if (item.tokenLen !== undefined) {
if (typeof item.tokenLen === 'number') {
hasTokens = true;
}
if (item.inputTokens !== undefined) {
if (typeof item.inputTokens === 'number') {
hasInputTokens = true;
}
if (item.outputTokens !== undefined) {
if (typeof item.outputTokens === 'number') {
hasOutputTokens = true;
}
if (item.textLen !== undefined) {
hasTextLen = true;
if (typeof item.charsLength === 'number') {
hasCharsLen = true;
}
if (item.duration !== undefined) {
if (typeof item.duration === 'number') {
hasDuration = true;
}
if (item.dataLen !== undefined) {
hasDataLen = true;
}
});
return {
@@ -71,7 +68,7 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
hasTokens,
hasInputTokens,
hasOutputTokens,
hasTextLen,
hasCharsLen,
hasDuration,
hasDataLen
};
@@ -123,9 +120,8 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
{hasTokens && <Th>{t('wallet.bill.Token Length')}</Th>}
{hasInputTokens && <Th>{t('wallet.bill.Input Token Length')}</Th>}
{hasOutputTokens && <Th>{t('wallet.bill.Output Token Length')}</Th>}
{hasTextLen && <Th>{t('wallet.bill.Text Length')}</Th>}
{hasCharsLen && <Th>{t('wallet.bill.Text Length')}</Th>}
{hasDuration && <Th>{t('wallet.bill.Duration')}</Th>}
{hasDataLen && <Th>{t('wallet.bill.Data Length')}</Th>}
<Th>费用(￥)</Th>
</Tr>
</Thead>
@@ -137,10 +133,8 @@ const BillDetail = ({ bill, onClose }: { bill: BillItemType; onClose: () => void
{hasTokens && <Td>{item.tokenLen ?? '-'}</Td>}
{hasInputTokens && <Td>{item.inputTokens ?? '-'}</Td>}
{hasOutputTokens && <Td>{item.outputTokens ?? '-'}</Td>}
{hasTextLen && <Td>{item.textLen ?? '-'}</Td>}
{hasCharsLen && <Td>{item.charsLength ?? '-'}</Td>}
{hasDuration && <Td>{item.duration ?? '-'}</Td>}
{hasDataLen && <Td>{item.dataLen ?? '-'}</Td>}
<Td>{formatStorePrice2Read(item.amount)}</Td>
</Tr>
))}

4
projects/app/src/pages/account/components/Info.tsx
View File

@@ -108,12 +108,12 @@ const UserInfo = () => {
});
} catch (err: any) {
toast({
title: typeof err === 'string' ? err : '头像选择异常',
title: typeof err === 'string' ? err : t('common.error.Select avatar failed'),
status: 'warning'
});
}
},
[onclickSave, toast, userInfo]
[onclickSave, t, toast, userInfo]
);
useQuery(['init'], initUserInfo, {

184
projects/app/src/pages/api/admin/initv46-2.ts
View File

@@ -1,184 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { defaultQAModels } from '@fastgpt/global/core/ai/model';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
try {
await Promise.allSettled([
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN data_id VARCHAR(50);`),
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN q DROP NOT NULL;`), // q can null
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN a DROP NOT NULL;`), // a can null
PgClient.query(
`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id TYPE VARCHAR(50) USING team_id::VARCHAR(50);`
), // team_id varchar
PgClient.query(
`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id TYPE VARCHAR(50) USING tmb_id::VARCHAR(50);`
), // tmb_id varchar
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN team_id SET NOT NULL;`), // team_id not null
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id SET NOT NULL;`), // tmb_id not null
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN dataset_id SET NOT NULL;`), // dataset_id not null
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN collection_id SET NOT NULL;`) // collection_id not null
]);
} catch (error) {}
try {
await initPgData();
} catch (error) {}
await MongoDataset.updateMany(
{},
{
agentModel: defaultQAModels[0].model
}
);
jsonRes(res, {
data: await init(limit),
message:
'初始化任务已开始，请注意日志进度。可通过 select count(id) from modeldata where data_id is null; 检查是否完全初始化，如果结果为 0 ，则完全初始化。'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
type PgItemType = {
id: string;
q: string;
a: string;
dataset_id: string;
collection_id: string;
team_id: string;
tmb_id: string;
};
async function initPgData() {
const limit = 10;
const { rows } = await PgClient.query<{ user_id: string }>(`
SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id='null';
`);
console.log('init pg', rows.length);
let success = 0;
for (let i = 0; i < limit; i++) {
init(i);
}
async function init(index: number): Promise<any> {
const userId = rows[index]?.user_id;
if (!userId) return;
try {
const tmb = await getUserDefaultTeam({ userId });
console.log(tmb);
// update pg
await PgClient.query(
`Update ${PgDatasetTableName} set team_id = '${String(tmb.teamId)}', tmb_id = '${String(
tmb.tmbId
)}' where user_id = '${userId}' AND team_id='null';`
);
console.log(++success);
init(index + limit);
} catch (error) {
if (error === 'default team not exist') {
return;
}
console.log(error);
await delay(1000);
return init(index);
}
}
}
async function init(limit: number): Promise<any> {
const { rows: idList } = await PgClient.query<{ id: string }>(
`SELECT id FROM ${PgDatasetTableName} WHERE data_id IS NULL`
);
console.log('totalCount', idList.length);
if (idList.length === 0) return;
for (let i = 0; i < limit; i++) {
initData(i);
}
async function initData(index: number): Promise<any> {
const dataId = idList[index]?.id;
if (!dataId) {
console.log('done');
return;
}
// get limit data where data_id is null
const { rows } = await PgClient.query<PgItemType>(
`SELECT id,q,a,dataset_id,collection_id,team_id,tmb_id FROM ${PgDatasetTableName} WHERE id=${dataId};`
);
const data = rows[0];
if (!data) {
console.log('done');
return;
}
let id = '';
try {
// create mongo data and update data_id
const { _id } = await MongoDatasetData.create({
teamId: data.team_id.trim(),
tmbId: data.tmb_id.trim(),
datasetId: data.dataset_id,
collectionId: data.collection_id,
q: data.q,
a: data.a,
fullTextToken: '',
indexes: [
{
defaultIndex: !data.a,
type: data.a ? DatasetDataIndexTypeEnum.qa : DatasetDataIndexTypeEnum.chunk,
dataId: data.id,
text: data.q
}
]
});
id = _id;
// update pg data_id
await PgClient.query(
`UPDATE ${PgDatasetTableName} SET data_id='${String(_id)}' WHERE id=${dataId};`
);
console.log(++success);
return initData(index + limit);
} catch (error) {
console.log(error);
console.log(data);
try {
if (id) {
await MongoDatasetData.findByIdAndDelete(id);
}
} catch (error) {}
await delay(500);
return initData(index);
}
}
}

173
projects/app/src/pages/api/admin/initv46-fix.ts
View File

@@ -1,173 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { Types, connectionMongo } from '@fastgpt/service/common/mongo';
import { TeamMemberCollectionName } from '@fastgpt/global/support/user/team/constant';
import { getUserDefaultTeam } from '@fastgpt/service/support/user/team/controller';
import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
await init(limit);
await initCollectionFileTeam(limit);
jsonRes(res, {});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
type PgItemType = {
id: string;
q: string;
a: string;
dataset_id: string;
collection_id: string;
data_id: string;
};
async function init(limit: number): Promise<any> {
const { rows } = await PgClient.query<{ id: string; data_id: string }>(
`SELECT id,data_id FROM ${PgDatasetTableName} WHERE team_id = tmb_id`
);
console.log('totalCount', rows.length);
await delay(2000);
if (rows.length === 0) return;
for (let i = 0; i < limit; i++) {
initData(i);
}
async function initData(index: number): Promise<any> {
const item = rows[index];
if (!item) {
console.log('done');
return;
}
// get mongo
const mongoData = await MongoDatasetData.findById(item.data_id, '_id teamId tmbId');
if (!mongoData) {
return initData(index + limit);
}
try {
// find team owner
const db = connectionMongo?.connection?.db;
const TeamMember = db.collection(TeamMemberCollectionName);
const tmb = await TeamMember.findOne({
teamId: new Types.ObjectId(mongoData.teamId),
role: 'owner'
});
if (!tmb) {
return initData(index + limit);
}
// update mongo and pg tmb_id
await MongoDatasetData.findByIdAndUpdate(item.data_id, {
tmbId: tmb._id
});
await PgClient.query(
`UPDATE ${PgDatasetTableName} SET tmb_id = '${String(tmb._id)}' WHERE id = '${item.id}'`
);
console.log(++success);
return initData(index + limit);
} catch (error) {
console.log(error);
await delay(500);
return initData(index);
}
}
}
async function initCollectionFileTeam(limit: number) {
/* init user default Team */
const DatasetFile = getGFSCollection('dataset');
const matchWhere = {
$or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }]
};
const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([
{
$match: matchWhere
},
{
$group: {
_id: '$metadata.userId', // 按 metadata.userId 分组以去重
userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId
}
},
{
$project: {
_id: 0, // 不显示 _id 字段
userId: 1 // 只显示 userId 字段
}
}
]).toArray();
const users = uniqueUsersWithNoTeamId;
console.log('un init total', users.length);
// limit 组一次
const userArr: any[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const item of userArr) {
await Promise.all(item.map((item) => init(item.userId)));
success += limit;
console.log(success);
}
async function init(userId: string): Promise<any> {
try {
const tmb = await getUserDefaultTeam({
userId
});
await DatasetFile.updateMany(
{
'metadata.userId': String(userId),
...matchWhere
},
{
$set: {
'metadata.teamId': String(tmb.teamId),
'metadata.tmbId': String(tmb.tmbId)
}
}
);
} catch (error) {
if (error === 'team not exist' || error === 'tmbId or userId is required') {
return;
}
console.log(error);
await delay(1000);
return init(userId);
}
}
}

330
projects/app/src/pages/api/admin/initv46.ts
View File

@@ -1,330 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoBill } from '@fastgpt/service/support/wallet/bill/schema';
import {
createDefaultTeam,
getUserDefaultTeam
} from '@fastgpt/service/support/user/team/controller';
import { MongoUser } from '@fastgpt/service/support/user/schema';
import { UserModelSchema } from '@fastgpt/global/support/user/type';
import { delay } from '@fastgpt/global/common/system/utils';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
import { POST } from '@fastgpt/service/common/api/plusRequest';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller';
import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50, maxSize = 3 } = req.body as { limit: number; maxSize: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
await initDefaultTeam(limit, maxSize);
await initMongoTeamId(limit);
await initDatasetAndApp();
await initCollectionFileTeam(limit);
if (FastGPTProUrl) {
POST('/admin/init46');
}
await initPgData();
jsonRes(res, {
data: {}
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
async function initDefaultTeam(limit: number, maxSize: number) {
/* init user default Team */
const users = await MongoUser.find({}, '_id balance');
console.log('init user default team', users.length);
// 100 组一次
const userArr: UserModelSchema[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const users of userArr) {
await Promise.all(users.map(init));
success += limit;
console.log(success);
}
async function init(user: UserModelSchema): Promise<any> {
try {
await createDefaultTeam({
userId: user._id,
balance: user.balance,
maxSize
});
} catch (error) {
console.log(error);
await delay(1000);
return init(user);
}
}
}
async function initMongoTeamId(limit: number) {
const mongoSchema = [
{
label: 'MongoPlugin',
schema: MongoPlugin
},
{
label: 'MongoChat',
schema: MongoChat
},
{
label: 'MongoChatItem',
schema: MongoChatItem
},
{
label: 'MongoApp',
schema: MongoApp
},
{
label: 'MongoDataset',
schema: MongoDataset
},
{
label: 'MongoDatasetCollection',
schema: MongoDatasetCollection
},
{
label: 'MongoDatasetTraining',
schema: MongoDatasetTraining
},
{
label: 'MongoBill',
schema: MongoBill
},
{
label: 'MongoOutLink',
schema: MongoOutLink
},
{
label: 'MongoOpenApi',
schema: MongoOpenApi
}
];
/* init user default Team */
for await (const item of mongoSchema) {
console.log('start init', item.label);
await initTeamTmbId(item.schema);
console.log('finish init', item.label);
}
async function initTeamTmbId(schema: any) {
const emptyWhere = {
$or: [{ teamId: { $exists: false } }, { teamId: null }]
};
const uniqueUsersWithNoTeamId = await schema.aggregate([
{
$match: emptyWhere
},
{
$group: {
_id: '$userId', // 按 userId 分组以去重
userId: { $first: '$userId' } // 保留第一个出现的 userId
}
},
{
$project: {
_id: 0, // 不显示 _id 字段
userId: 1 // 只显示 userId 字段
}
}
]);
const users = uniqueUsersWithNoTeamId;
console.log('un init total', users.length);
// limit 组一次
const userArr: any[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const users of userArr) {
await Promise.all(users.map((item) => init(item.userId)));
success += limit;
console.log(success);
}
async function init(userId: string): Promise<any> {
try {
const tmb = await getUserDefaultTeam({ userId });
await schema.updateMany(
{
userId,
...emptyWhere
},
{
teamId: tmb.teamId,
tmbId: tmb.tmbId
}
);
} catch (error) {
if (error === 'team not exist' || error === 'tmbId or userId is required') {
return;
}
console.log(error);
await delay(1000);
return init(userId);
}
}
}
}
async function initDatasetAndApp() {
await MongoDataset.updateMany(
{},
{
$set: {
permission: PermissionTypeEnum.private
}
}
);
await MongoApp.updateMany(
{},
{
$set: {
permission: PermissionTypeEnum.private
}
}
);
}
async function initCollectionFileTeam(limit: number) {
/* init user default Team */
const DatasetFile = getGFSCollection('dataset');
const matchWhere = {
$or: [{ 'metadata.teamId': { $exists: false } }, { 'metadata.teamId': null }]
};
const uniqueUsersWithNoTeamId = await DatasetFile.aggregate([
{
$match: matchWhere
},
{
$group: {
_id: '$metadata.userId', // 按 metadata.userId 分组以去重
userId: { $first: '$metadata.userId' } // 保留第一个出现的 userId
}
},
{
$project: {
_id: 0, // 不显示 _id 字段
userId: 1 // 只显示 userId 字段
}
}
]).toArray();
const users = uniqueUsersWithNoTeamId;
console.log('un init total', users.length);
// limit 组一次
const userArr: any[][] = [];
for (let i = 0; i < users.length; i += limit) {
userArr.push(users.slice(i, i + limit));
}
let success = 0;
for await (const item of userArr) {
await Promise.all(item.map((item) => init(item.userId)));
success += limit;
console.log(success);
}
async function init(userId: string): Promise<any> {
try {
const tmb = await getUserDefaultTeam({
userId
});
await DatasetFile.updateMany(
{
'metadata.userId': String(userId),
...matchWhere
},
{
$set: {
'metadata.teamId': String(tmb.teamId),
'metadata.tmbId': String(tmb.tmbId)
}
}
);
} catch (error) {
if (error === 'team not exist' || error === 'tmbId or userId is required') {
return;
}
console.log(error);
await delay(1000);
return init(userId);
}
}
}
async function initPgData() {
const limit = 10;
// add column
try {
await Promise.allSettled([
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN team_id VARCHAR(50);`),
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ADD COLUMN tmb_id VARCHAR(50);`),
PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN user_id DROP NOT NULL;`)
]);
} catch (error) {
console.log(error);
console.log('column exists');
}
const { rows } = await PgClient.query<{ user_id: string }>(`
SELECT DISTINCT user_id FROM ${PgDatasetTableName} WHERE team_id IS NULL;
`);
console.log('init pg', rows.length);
let success = 0;
for (let i = 0; i < limit; i++) {
init(i);
}
async function init(index: number): Promise<any> {
const userId = rows[index]?.user_id;
if (!userId) return;
try {
const tmb = await getUserDefaultTeam({ userId });
// update pg
await PgClient.query(
`Update ${PgDatasetTableName} set team_id = '${tmb.teamId}', tmb_id = '${tmb.tmbId}' where user_id = '${userId}' AND team_id IS NULL;`
);
console.log(++success);
init(index + limit);
} catch (error) {
if (error === 'default team not exist') {
return;
}
console.log(error);
await delay(1000);
return init(index);
}
}
}

2
projects/app/src/pages/api/admin/initv463.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetStatusEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
let success = 0;

106
projects/app/src/pages/api/admin/initv467.ts Normal file
View File

@@ -0,0 +1,106 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { MongoImage } from '@fastgpt/service/common/file/image/schema';
import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type';
import { delay } from '@fastgpt/global/common/system/utils';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { getNanoid } from '@fastgpt/global/common/string/tools';
let success = 0;
let deleteImg = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { test = false } = req.body as { test: boolean };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
deleteImg = 0;
// 取消 pg tmb_id 和 data_id 的null
await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN tmb_id DROP NOT NULL;`);
await PgClient.query(`ALTER TABLE ${PgDatasetTableName} ALTER COLUMN data_id DROP NOT NULL;`);
// 重新绑定 images 和 collections
const images = await MongoImage.find(
{ 'metadata.fileId': { $exists: true } },
'_id metadata'
).lean();
// 去除 fileId 相同的数据
const fileIdMap = new Map<string, MongoImageSchemaType>();
images.forEach((image) => {
// @ts-ignore
const fileId = image.metadata?.fileId;
if (!fileIdMap.has(fileId) && fileId) {
fileIdMap.set(fileId, image);
}
});
const images2 = Array.from(fileIdMap.values());
console.log('total image list', images2.length);
for await (const image of images2) {
await initImages(image, test);
}
jsonRes(res, {
data: success,
message: 'success'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
export const initImages = async (image: MongoImageSchemaType, test: boolean): Promise<any> => {
try {
//@ts-ignore
const fileId = image.metadata.fileId as string;
if (!fileId) return;
// 找到集合
const collection = await MongoDatasetCollection.findOne({ fileId }, '_id metadata').lean();
if (!collection) {
deleteImg++;
console.log('deleteImg', deleteImg);
if (test) return;
return MongoImage.deleteOne({ _id: image._id });
}
const relatedImageId = getNanoid(24);
// update image
if (!test) {
await Promise.all([
MongoImage.updateMany(
{ 'metadata.fileId': fileId },
{ $set: { 'metadata.relatedId': relatedImageId } }
),
MongoDatasetCollection.findByIdAndUpdate(collection._id, {
$set: {
'metadata.relatedImgId': relatedImageId
}
})
]);
}
success++;
console.log('success', success);
} catch (error) {
console.log(error);
await delay(1000);
return initImages(image, test);
}
};

95
projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts
View File

@@ -1,95 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import {
delFileByFileIdList,
getGFSCollection
} from '@fastgpt/service/common/file/gridfs/controller';
import { addLog } from '@fastgpt/service/common/system/log';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { delay } from '@fastgpt/global/common/system/utils';
/*
check dataset.files data. If there is no match in dataset.collections, delete it
*/
let deleteFileAmount = 0;
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const {
startDay = 10,
endDay = 3,
limit = 30
} = req.body as { startDay?: number; endDay?: number; limit?: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
// start: now - maxDay, end: now - 3 day
const start = new Date(Date.now() - startDay * 24 * 60 * 60 * 1000);
const end = new Date(Date.now() - endDay * 24 * 60 * 60 * 1000);
deleteFileAmount = 0;
checkFiles(start, end, limit);
jsonRes(res, {
message: 'success'
});
} catch (error) {
addLog.error(`check valid dataset files error`, error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function checkFiles(start: Date, end: Date, limit: number) {
const collection = getGFSCollection('dataset');
const where = {
uploadDate: { $gte: start, $lte: end }
};
// 1. get all _id
const ids = await collection
.find(where, {
projection: {
_id: 1
}
})
.toArray();
console.log('total files', ids.length);
for (let i = 0; i < limit; i++) {
check(i);
}
async function check(index: number): Promise<any> {
const id = ids[index];
if (!id) {
console.log(`检测完成，共删除 ${deleteFileAmount} 个无效文件`);
return;
}
try {
const { _id } = id;
// 2. find fileId in dataset.collections
const hasCollection = await MongoDatasetCollection.countDocuments({ fileId: _id });
// 3. if not found, delete file
if (hasCollection === 0) {
await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(_id)] });
console.log('delete file', _id);
deleteFileAmount++;
}
index % 100 === 0 && console.log(index);
return check(index + limit);
} catch (error) {
console.log(error);
await delay(2000);
return check(index);
}
}
}

30
projects/app/src/pages/api/common/file/upload.ts
View File

@@ -18,32 +18,24 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
try {
const { userId, teamId, tmbId } = await authCert({ req, authToken: true });
const { files, bucketName, metadata } = await upload.doUpload(req, res);
filePaths = files.map((file) => file.path);
const { file, bucketName, metadata } = await upload.doUpload(req, res);
filePaths = [file.path];
await connectToDatabase();
if (!bucketName) {
throw new Error('bucketName is empty');
}
const upLoadResults = await Promise.all(
files.map((file) =>
uploadFile({
teamId,
tmbId,
bucketName,
path: file.path,
filename: file.originalname,
metadata: {
...metadata,
contentType: file.mimetype,
userId
}
})
)
);
const upLoadResults = await uploadFile({
teamId,
tmbId,
bucketName,
path: file.path,
filename: file.originalname,
contentType: file.mimetype,
metadata: metadata
});
jsonRes(res, {
data: upLoadResults

3
projects/app/src/pages/api/common/system/getInitData.ts
View File

@@ -55,7 +55,8 @@ const defaultFeConfigs: FastGPTFeConfigsType = {
websiteSyncLimitMinuted: 0
},
scripts: [],
favicon: '/favicon.ico'
favicon: '/favicon.ico',
uploadFileMaxSize: 500
};
export async function getInitConfig() {

40
projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
View File

@@ -7,7 +7,7 @@ import { jsonRes } from '@fastgpt/service/common/response';
import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d';
import type { ModuleItemType } from '@fastgpt/global/core/module/type';
import { FormatForm2ModulesProps } from '@fastgpt/global/core/app/api';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import { getExtractModel } from '@/service/core/ai/model';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -37,7 +37,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
return [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -49,7 +49,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -58,7 +58,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -93,7 +93,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -189,7 +189,7 @@ function simpleChatTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -268,7 +268,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
const modules: ModuleItemType[] = [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -280,7 +280,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -289,17 +289,13 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
{
moduleId: 'vuc92c',
key: 'userChatInput'
},
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
@@ -307,7 +303,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
},
{
moduleId: 'datasetSearch',
name: '知识库搜索',
name: 'core.module.template.Dataset search',
avatar: '/imgs/module/db.png',
flowType: 'datasetSearchNode',
showStatus: true,
@@ -447,6 +443,18 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
valueType: 'boolean',
type: 'source',
targets: []
},
{
key: 'userChatInput',
label: 'core.module.input.label.user question',
type: 'hidden',
valueType: 'string',
targets: [
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
]
},
@@ -473,7 +481,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -569,7 +577,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:

38
projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts
View File

@@ -33,7 +33,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
return [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -45,7 +45,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -54,7 +54,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
@@ -89,7 +89,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -185,7 +185,7 @@ function simpleChatTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:
@@ -264,7 +264,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
const modules: ModuleItemType[] = [
{
moduleId: 'userChatInput',
name: '用户问题(对话入口)',
name: 'core.module.template.Chat entrance',
avatar: '/imgs/module/userChatInput.png',
flowType: 'questionInput',
position: {
@@ -276,7 +276,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
key: 'userChatInput',
type: 'systemInput',
valueType: 'string',
label: '用户问题',
label: 'core.module.input.label.user question',
showTargetInApp: false,
showTargetInPlugin: false,
connected: false
@@ -285,17 +285,13 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
outputs: [
{
key: 'userChatInput',
label: '用户问题',
label: 'core.module.input.label.user question',
type: 'source',
valueType: 'string',
targets: [
{
moduleId: 'vuc92c',
key: 'userChatInput'
},
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
@@ -303,7 +299,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
},
{
moduleId: 'datasetSearch',
name: '知识库搜索',
name: 'core.module.template.Dataset search',
avatar: '/imgs/module/db.png',
flowType: 'datasetSearchNode',
showStatus: true,
@@ -457,6 +453,18 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
valueType: 'boolean',
type: 'source',
targets: []
},
{
key: 'userChatInput',
label: 'core.module.input.label.user question',
type: 'hidden',
valueType: 'string',
targets: [
{
moduleId: 'chatModule',
key: 'userChatInput'
}
]
}
]
},
@@ -483,7 +491,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'model',
type: 'selectChatModel',
label: '对话模型',
label: 'core.module.input.label.aiModel',
required: true,
valueType: 'string',
showTargetInApp: false,
@@ -579,7 +587,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
{
key: 'systemPrompt',
type: 'textarea',
label: '系统提示词',
label: 'core.ai.Prompt',
max: 300,
valueType: 'string',
description:

19
projects/app/src/pages/api/core/app/getChatLogs.ts
View File

@@ -8,6 +8,7 @@ import { Types } from '@fastgpt/service/common/mongo';
import { addDays } from 'date-fns';
import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { ChatItemCollectionName } from '@fastgpt/service/core/chat/chatItemSchema';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
@@ -28,8 +29,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
const { teamId } = await authApp({ req, authToken: true, appId, per: 'w' });
const where = {
appId: new Types.ObjectId(appId),
teamId: new Types.ObjectId(teamId),
appId: new Types.ObjectId(appId),
updateTime: {
$gte: new Date(dateStart),
$lte: new Date(dateEnd)
@@ -41,18 +42,26 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
{ $match: where },
{
$lookup: {
from: 'chatitems',
let: { chat_id: '$chatId' },
from: ChatItemCollectionName,
let: { chatId: '$chatId' },
pipeline: [
{
$match: {
$expr: {
$and: [
{ $eq: ['$chatId', '$$chat_id'] },
{ $eq: ['$appId', new Types.ObjectId(appId)] }
{ $eq: ['$appId', new Types.ObjectId(appId)] },
{ $eq: ['$chatId', '$$chatId'] }
]
}
}
},
{
$project: {
userGoodFeedback: 1,
userBadFeedback: 1,
customFeedbacks: 1,
adminFeedback: 1
}
}
],
as: 'chatitems'

3
projects/app/src/pages/api/core/chat/clearHistories.ts
View File

@@ -35,16 +35,17 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
return Promise.reject('Param are error');
})();
console.log(match);
// find chatIds
const list = await MongoChat.find(match, 'chatId').lean();
const idList = list.map((item) => item.chatId);
await MongoChatItem.deleteMany({
appId,
chatId: { $in: idList }
});
await MongoChat.deleteMany({
appId,
chatId: { $in: idList }
});

2
projects/app/src/pages/api/core/chat/delHistory.ts
View File

@@ -23,9 +23,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
await MongoChatItem.deleteMany({
appId,
chatId
});
await MongoChat.findOneAndRemove({
appId,
chatId
});

2
projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts
View File

@@ -27,6 +27,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await MongoChatItem.findOneAndUpdate(
{
appId,
chatId,
dataId: chatItemId
},
{

13
projects/app/src/pages/api/core/chat/feedback/closeCustom.ts
View File

@@ -2,14 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import type {
AdminUpdateFeedbackParams,
CloseCustomFeedbackParams
} from '@/global/core/chat/api.d';
import type { CloseCustomFeedbackParams } from '@/global/core/chat/api.d';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { autChatCrud } from '@/service/support/permission/auth/chat';
/* 初始化我的聊天框，需要身份验证 */
/* remove custom feedback */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
@@ -29,13 +26,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await authCert({ req, authToken: true });
await MongoChatItem.findOneAndUpdate(
{ dataId: chatItemId },
{ appId, chatId, dataId: chatItemId },
{ $unset: { [`customFeedbacks.${index}`]: 1 } }
);
await MongoChatItem.findOneAndUpdate(
{ dataId: chatItemId },
{ $pull: { customFeedbacks: null } }
);
jsonRes(res);
} catch (err) {

1
projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts
View File

@@ -29,6 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
await MongoChatItem.findOneAndUpdate(
{
appId,
chatId,
dataId: chatItemId
},

2
projects/app/src/pages/api/core/chat/getHistories.ts
View File

@@ -31,8 +31,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
if (appId) {
const { tmbId } = await authCert({ req, authToken: true });
return {
appId,
tmbId,
appId,
source: ChatSourceEnum.online
};
}

3
projects/app/src/pages/api/core/chat/init.ts
View File

@@ -31,7 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
appId,
per: 'r'
}),
chatId ? MongoChat.findOne({ chatId }) : undefined
chatId ? MongoChat.findOne({ appId, chatId }) : undefined
]);
// auth chat permission
@@ -41,6 +41,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// get app and history
const { history } = await getChatItems({
appId,
chatId,
limit: 30,
field: `dataId obj value adminFeedback userBadFeedback userGoodFeedback ${

5
projects/app/src/pages/api/core/chat/item/delete.ts
View File

@@ -25,8 +25,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
await MongoChatItem.deleteOne({
dataId: contentId,
chatId
appId,
chatId,
dataId: contentId
});
jsonRes(res);

2
projects/app/src/pages/api/core/chat/item/getSpeech.ts
View File

@@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
try {
pushAudioSpeechBill({
model: model,
textLen: input.length,
charsLength: input.length,
tmbId,
teamId,
source: authType2BillSource({ authType })

3
projects/app/src/pages/api/core/chat/outLink/init.ts
View File

@@ -26,7 +26,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
// auth app permission
const [tmb, chat, app] = await Promise.all([
MongoTeamMember.findById(shareChat.tmbId, '_id userId').populate('userId', 'avatar').lean(),
MongoChat.findOne({ chatId, shareId }).lean(),
MongoChat.findOne({ appId, chatId, shareId }).lean(),
MongoApp.findById(appId).lean()
]);
@@ -40,6 +40,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
}
const { history } = await getChatItems({
appId: app._id,
chatId,
limit: 30,
field: `dataId obj value userGoodFeedback userBadFeedback ${

2
projects/app/src/pages/api/core/chat/updateHistory.ts
View File

@@ -22,7 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
});
await MongoChat.findOneAndUpdate(
{ chatId },
{ appId, chatId },
{
...(customTitle !== undefined && { customTitle }),
...(top !== undefined && { top })

2
projects/app/src/pages/api/core/dataset/allDataset.ts
View File

@@ -6,7 +6,7 @@ import { getVectorModel } from '@/service/core/ai/model';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
/* get all dataset by teamId or tmbId */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

85
projects/app/src/pages/api/core/dataset/collection/create/file.ts Normal file
View File

@@ -0,0 +1,85 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller';
import { getUploadModel } from '@fastgpt/service/common/file/multer';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { FileCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api';
import { removeFilesByPaths } from '@fastgpt/service/common/file/utils';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
/**
* Creates the multer uploader
*/
const upload = getUploadModel({
maxSize: 500 * 1024 * 1024
});
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
let filePaths: string[] = [];
const { datasetId } = req.query as { datasetId: string };
try {
await connectToDatabase();
const { teamId, tmbId } = await authDataset({
req,
authToken: true,
authApiKey: true,
per: 'w',
datasetId
});
const { file, bucketName, data } = await upload.doUpload<FileCreateDatasetCollectionParams>(
req,
res
);
filePaths = [file.path];
if (!file || !bucketName) {
throw new Error('file is empty');
}
const { fileMetadata, collectionMetadata, ...collectionData } = data;
// upload file and create collection
const fileId = await uploadFile({
teamId,
tmbId,
bucketName,
path: file.path,
filename: file.originalname,
contentType: file.mimetype,
metadata: fileMetadata
});
// create collection
const collectionId = await createOneCollection({
...collectionData,
metadata: collectionMetadata,
teamId,
tmbId,
type: DatasetCollectionTypeEnum.file,
fileId
});
jsonRes(res, {
data: collectionId
});
} catch (error) {
jsonRes(res, {
code: 500,
error
});
}
removeFilesByPaths(filePaths);
}
export const config = {
api: {
bodyParser: false
}
};

5
projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts → projects/app/src/pages/api/core/dataset/collection/create/link.ts
View File

@@ -7,7 +7,10 @@ import { connectToDatabase } from '@/service/mongo';
import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';

12
projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts → projects/app/src/pages/api/core/dataset/collection/create/text.ts
View File

@@ -7,11 +7,14 @@ import { connectToDatabase } from '@/service/mongo';
import type { TextCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import {
TrainingModeEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller';
import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller';
import { hashStr } from '@fastgpt/global/common/string/tools';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -39,8 +42,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
text,
chunkLen: chunkSize,
overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0,
customReg: chunkSplitter ? [chunkSplitter] : [],
countTokens: false
customReg: chunkSplitter ? [chunkSplitter] : []
});
// 2. check dataset limit
@@ -67,7 +69,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// 4. push chunks to training queue
const insertResults = await pushDataToDatasetCollection({
const insertResults = await pushDataToTrainingQueue({
teamId,
tmbId,
collectionId,

17
projects/app/src/pages/api/core/dataset/collection/delete.ts
View File

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
@@ -15,7 +15,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('CollectionIdId is required');
}
await authDatasetCollection({
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
@@ -24,13 +24,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// find all delete id
const collections = await findCollectionAndChild(collectionId, '_id fileId');
const delIdList = collections.map((item) => item._id);
const collections = await findCollectionAndChild({
teamId,
datasetId: collection.datasetId._id,
collectionId,
fields: '_id teamId fileId metadata'
});
// delete
await delCollectionRelevantData({
collectionIds: delIdList,
fileIds: collections.map((item) => item?.fileId || '').filter(Boolean)
await delCollectionAndRelatedSources({
collections
});
jsonRes(res);

10
projects/app/src/pages/api/core/dataset/collection/list.ts
View File

@@ -7,7 +7,7 @@ import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.
import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
import { PagingData } from '@/types';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { startQueue } from '@/service/utils/tools';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema';
@@ -87,7 +87,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{
$match: {
$expr: {
$eq: ['$collectionId', '$$id']
$and: [{ $eq: ['$teamId', match.teamId] }, { $eq: ['$collectionId', '$$id'] }]
}
}
},
@@ -105,7 +105,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
{
$match: {
$expr: {
$eq: ['$collectionId', '$$id']
$and: [
{ $eq: ['$teamId', match.teamId] },
{ $eq: ['$datasetId', match.datasetId] },
{ $eq: ['$collectionId', '$$id'] }
]
}
}
},

11
projects/app/src/pages/api/core/dataset/collection/sync/link.ts
View File

@@ -6,11 +6,11 @@ import {
getCollectionAndRawText,
reloadCollectionChunks
} from '@fastgpt/service/core/dataset/collection/utils';
import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { delCollectionAndRelatedSources } from '@fastgpt/service/core/dataset/collection/controller';
import {
DatasetCollectionSyncResultEnum,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constant';
} from '@fastgpt/global/core/dataset/constants';
import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
@@ -27,7 +27,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
throw new Error('CollectionIdId is required');
}
const { collection, tmbId } = await authDatasetCollection({
const { collection, teamId, tmbId } = await authDatasetCollection({
req,
authToken: true,
collectionId,
@@ -87,9 +87,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
// delete old collection
await delCollectionRelevantData({
collectionIds: [collection._id],
fileIds: collection.fileId ? [collection.fileId] : []
await delCollectionAndRelatedSources({
collections: [collection]
});
jsonRes(res, {

2
projects/app/src/pages/api/core/dataset/create.ts
View File

@@ -5,7 +5,7 @@ import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import { createDefaultCollection } from '@fastgpt/service/core/dataset/collection/controller';
import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {

20
projects/app/src/pages/api/core/dataset/data/delete.ts
View File

@@ -3,7 +3,8 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { connectToDatabase } from '@/service/mongo';
import { authDatasetData } from '@/service/support/permission/auth/dataset';
import { delDatasetDataByDataId } from '@fastgpt/service/core/dataset/data/controller';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { deleteDatasetDataVector } from '@fastgpt/service/common/vectorStore/controller';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -17,7 +18,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
}
// 凭证校验
const { datasetData } = await authDatasetData({
const { teamId, datasetData } = await authDatasetData({
req,
authToken: true,
authApiKey: true,
@@ -25,11 +26,20 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
per: 'w'
});
await delDatasetDataByDataId({
collectionId: datasetData.collectionId,
mongoDataId: dataId
// update mongo data update time
await MongoDatasetData.findByIdAndUpdate(dataId, {
updateTime: new Date()
});
// delete vector data
await deleteDatasetDataVector({
teamId,
idList: datasetData.indexes.map((item) => item.dataId)
});
// delete mongo data
await MongoDatasetData.findByIdAndDelete(dataId);
jsonRes(res, {
data: 'success'
});

5
projects/app/src/pages/api/core/dataset/data/insertData.ts
View File

@@ -71,12 +71,13 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// Duplicate data check
await hasSameValue({
teamId,
collectionId,
q: formatQ,
a: formatA
});
const { insertId, tokens } = await insertData2Dataset({
const { insertId, charsLength } = await insertData2Dataset({
teamId,
tmbId,
datasetId,
@@ -91,7 +92,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model: vectorModelData.model
});

10
projects/app/src/pages/api/core/dataset/data/list.ts
View File

@@ -20,11 +20,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
pageSize = Math.min(pageSize, 30);
// 凭证校验
await authDatasetCollection({ req, authToken: true, authApiKey: true, collectionId, per: 'r' });
const { teamId, collection } = await authDatasetCollection({
req,
authToken: true,
authApiKey: true,
collectionId,
per: 'r'
});
searchText = searchText.replace(/'/g, '');
const match = {
teamId,
datasetId: collection.datasetId._id,
collectionId,
...(searchText
? {

5
projects/app/src/pages/api/core/dataset/data/pushData.ts
View File

@@ -3,13 +3,12 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant';
import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
import type { PushDatasetDataProps } from '@/global/core/dataset/api.d';
import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils';
import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller';
import { pushDataToTrainingQueue } from '@/service/core/dataset/data/controller';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -41,7 +40,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
jsonRes<PushDataResponse>(res, {
data: await pushDataToDatasetCollection({
data: await pushDataToTrainingQueue({
...req.body,
teamId,
tmbId

4
projects/app/src/pages/api/core/dataset/data/update.ts
View File

@@ -31,7 +31,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// auth team balance
await authTeamBalance(teamId);
const { tokens } = await updateData2Dataset({
const { charsLength } = await updateData2Dataset({
dataId: id,
q,
a,
@@ -42,7 +42,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model: vectorModel
});

19
projects/app/src/pages/api/core/dataset/delete.ts
View File

@@ -2,32 +2,35 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/data/controller';
import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller';
import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { id } = req.query as {
const { id: datasetId } = req.query as {
id: string;
};
if (!id) {
if (!datasetId) {
throw new Error('缺少参数');
}
// auth owner
await authDataset({ req, authToken: true, datasetId: id, per: 'owner' });
const { teamId } = await authDataset({ req, authToken: true, datasetId, per: 'owner' });
const deletedIds = await findDatasetIdTreeByTopDatasetId(id);
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId
});
// delete all dataset.data and pg data
await delDatasetRelevantData({ datasetIds: deletedIds });
await delDatasetRelevantData({ datasets });
// delete dataset data
await MongoDataset.deleteMany({
_id: { $in: deletedIds }
_id: { $in: datasets.map((d) => d._id) }
});
jsonRes(res);

11
projects/app/src/pages/api/core/dataset/exportAll.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { addLog } from '@fastgpt/service/common/system/log';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { findDatasetIdTreeByTopDatasetId } from '@fastgpt/service/core/dataset/controller';
import { findDatasetAndAllChildren } from '@fastgpt/service/core/dataset/controller';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import {
checkExportDatasetLimit,
@@ -30,7 +30,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
limitMinutes: global.feConfigs?.limit?.exportDatasetLimitMinutes
});
const exportIds = await findDatasetIdTreeByTopDatasetId(datasetId);
const datasets = await findDatasetAndAllChildren({
teamId,
datasetId,
fields: '_id'
});
res.setHeader('Content-Type', 'text/csv; charset=utf-8;');
res.setHeader('Content-Disposition', 'attachment; filename=dataset.csv; ');
@@ -42,7 +46,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
a: string;
}>(
{
datasetId: { $in: exportIds }
teamId,
datasetId: { $in: datasets.map((d) => d._id) }
},
'q a'
)

2
projects/app/src/pages/api/core/dataset/list.ts
View File

@@ -2,7 +2,7 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import type { DatasetListItemType } from '@fastgpt/global/core/dataset/type.d';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { mongoRPermission } from '@fastgpt/global/support/permission/utils';
import { authUserRole } from '@fastgpt/service/support/permission/auth/user';

5
projects/app/src/pages/api/core/dataset/searchTest.ts
View File

@@ -47,7 +47,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// model: global.chatModels[0].model
// });
const { searchRes, tokens, ...result } = await searchDatasetData({
const { searchRes, charsLength, ...result } = await searchDatasetData({
teamId,
rawQuery: text,
queries: [text],
model: dataset.vectorModel,
@@ -62,7 +63,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model: dataset.vectorModel,
source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt
});

2
projects/app/src/pages/api/plugins/customFeedback/index.ts
View File

@@ -12,6 +12,7 @@ type Props = HttpBodyType<{
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
const {
appId,
chatId,
responseChatItemId: chatItemId,
data: { defaultFeedback, customFeedback }
@@ -30,6 +31,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
// wait the chat finish
setTimeout(() => {
addCustomFeedbacks({
appId,
chatId,
chatItemId,
feedbacks: [feedback]

11
projects/app/src/pages/api/support/user/account/loginByPassword.ts
View File

@@ -10,7 +10,7 @@ import { UserStatusEnum } from '@fastgpt/global/support/user/constant';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { username, password, tmbId = '' } = req.body as PostLoginProps;
const { username, password } = req.body as PostLoginProps;
if (!username || !password) {
throw new Error('缺少参数');
@@ -40,7 +40,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
throw new Error('密码错误');
}
const userDetail = await getUserDetail({ tmbId, userId: user._id });
const userDetail = await getUserDetail({
tmbId: user?.lastLoginTmbId,
userId: user._id
});
MongoUser.findByIdAndUpdate(user._id, {
lastLoginTmbId: userDetail.team.tmbId
});
const token = createJWT(userDetail);
setCookie(res, token);

37
projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts Normal file
View File

@@ -0,0 +1,37 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { size } = req.query as {
size: string;
};
// 凭证校验
const { teamId } = await authCert({ req, authToken: true });
if (!size) {
return jsonRes(res);
}
const numberSize = Number(size);
await checkDatasetLimit({
teamId,
freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize,
insertLen: numberSize
});
jsonRes(res);
} catch (err) {
res.status(500);
jsonRes(res, {
code: 500,
error: err
});
}
}

19
projects/app/src/pages/api/support/wallet/bill/createTrainingBill.ts
View File

@@ -1,29 +1,32 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { CreateTrainingBillProps } from '@fastgpt/global/support/wallet/bill/api.d';
import { getQAModel, getVectorModel } from '@/service/core/ai/model';
import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
const { name, vectorModel, agentModel } = req.body as CreateTrainingBillProps;
const { name, datasetId } = req.body as CreateTrainingBillProps;
const { teamId, tmbId } = await authCert({ req, authToken: true, authApiKey: true });
const vectorModelData = getVectorModel(vectorModel);
const agentModelData = getQAModel(agentModel);
const { teamId, tmbId, dataset } = await authDataset({
req,
authToken: true,
authApiKey: true,
datasetId,
per: 'w'
});
const { billId } = await createTrainingBill({
teamId,
tmbId,
appName: name,
billSource: BillSourceEnum.training,
vectorModel: vectorModelData.name,
agentModel: agentModelData.name
vectorModel: getVectorModel(dataset.vectorModel).name,
agentModel: getQAModel(dataset.agentModel).name
});
jsonRes<string>(res, {

8
projects/app/src/pages/api/v1/audio/transcriptions.ts
View File

@@ -17,11 +17,11 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
try {
const {
files,
metadata: { duration, shareId }
file,
data: { duration }
} = await upload.doUpload<{ duration: number; shareId?: string }>(req, res);
filePaths = files.map((file) => file.path);
filePaths = [file.path];
const { teamId, tmbId } = await authCert({ req, authToken: true });
@@ -29,8 +29,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
throw new Error('whisper model not found');
}
const file = files[0];
if (!file) {
throw new Error('file not found');
}

7
projects/app/src/pages/api/v1/chat/completions.ts
View File

@@ -195,7 +195,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
});
// get and concat history
const { history } = await getChatItems({ chatId, limit: 30, field: `dataId obj value` });
const { history } = await getChatItems({
appId: app._id,
chatId,
limit: 30,
field: `dataId obj value`
});
const concatHistories = history.concat(chatMessages);
const responseChatItemId: string | undefined = messages[messages.length - 1].dataId;

8
projects/app/src/pages/api/v1/embeddings.ts
View File

@@ -33,7 +33,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
await authTeamBalance(teamId);
const { tokens, vectors } = await getVectorsByText({ input: query, model });
const { charsLength, vectors } = await getVectorsByText({ input: query, model });
res.json({
object: 'list',
@@ -44,15 +44,15 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
})),
model,
usage: {
prompt_tokens: tokens,
total_tokens: tokens
prompt_tokens: charsLength,
total_tokens: charsLength
}
});
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokens,
charsLength,
model,
billId,
source: getBillSourceByAuthType({ authType })

45
projects/app/src/pages/app/detail/components/FlowEdit/Header.tsx
View File

@@ -62,7 +62,7 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({
});
if (unconnected) {
const msg = `【${t(item.name)}】存在未填或未连接参数`;
const msg = t('core.module.Unlink tip', { name: t(item.name) });
toast({
status: 'warning',
@@ -82,8 +82,8 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({
permission: undefined
});
},
successToast: '保存配置成功',
errorToast: '保存配置异常',
successToast: t('common.Save Success'),
errorToast: t('common.Save Failed'),
onSuccess() {
ChatTestRef.current?.resetChatTest();
}
@@ -98,22 +98,23 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({
alignItems={'center'}
userSelect={'none'}
>
<MyTooltip label={t('common.Back')} offset={[10, 10]}>
<IconButton
size={'smSquare'}
icon={<MyIcon name={'common/backLight'} w={'14px'} />}
borderColor={'myGray.300'}
variant={'whiteBase'}
aria-label={''}
onClick={openConfirmOut(async () => {
const modules = await flow2ModulesAndCheck();
if (modules) {
await onclickSave(modules);
}
onClose();
}, onClose)}
/>
</MyTooltip>
<IconButton
size={'smSquare'}
icon={<MyIcon name={'common/backFill'} w={'14px'} />}
borderRadius={'50%'}
w={'26px'}
h={'26px'}
borderColor={'myGray.300'}
variant={'whiteBase'}
aria-label={''}
onClick={openConfirmOut(async () => {
const modules = await flow2ModulesAndCheck();
if (modules) {
await onclickSave(modules);
}
onClose();
}, onClose)}
/>
<Box ml={[3, 6]} fontSize={['md', '2xl']} flex={1}>
{app.name}
</Box>
@@ -154,7 +155,7 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({
onClick={() => setTestModules(undefined)}
/>
) : (
<MyTooltip label={'测试对话'}>
<MyTooltip label={t('core.Chat test')}>
<IconButton
mr={[3, 6]}
icon={<MyIcon name={'core/chat/chatLight'} w={['14px', '16px']} />}
@@ -171,9 +172,9 @@ const RenderHeaderContainer = React.memo(function RenderHeaderContainer({
</MyTooltip>
)}
<MyTooltip label={'保存配置'}>
<MyTooltip label={t('common.Save')}>
<IconButton
icon={<MyIcon name={'save'} w={['14px', '16px']} />}
icon={<MyIcon name={'common/saveFill'} w={['14px', '16px']} />}
size={'smSquare'}
isLoading={isLoading}
aria-label={'save'}

10
projects/app/src/pages/app/detail/components/FlowEdit/index.tsx
View File

@@ -44,12 +44,18 @@ const Render = ({ app, onClose }: Props) => {
initData(JSON.parse(JSON.stringify(app.modules)));
}, [app.modules]);
return <Flow templates={moduleTemplates} Header={<Header app={app} onClose={onClose} />} />;
const memoRender = useMemo(() => {
return <Flow templates={moduleTemplates} Header={<Header app={app} onClose={onClose} />} />;
}, [app, moduleTemplates.length, onClose]);
return memoRender;
};
export default React.memo(function FlowEdit(props: Props) {
const filterAppIds = useMemo(() => [props.app._id], [props.app._id]);
return (
<FlowProvider mode={'app'} filterAppIds={[props.app._id]}>
<FlowProvider mode={'app'} filterAppIds={filterAppIds}>
<Render {...props} />
</FlowProvider>
);

31
projects/app/src/pages/app/detail/components/InfoModal.tsx
View File

@@ -52,7 +52,7 @@ const InfoModal = ({
});
const [refresh, setRefresh] = useState(false);
// 提交保存模型修改
// submit config
const { mutate: saveSubmitSuccess, isLoading: btnLoading } = useRequest({
mutationFn: async (data: AppSchema) => {
await updateAppDetail(data._id, {
@@ -66,18 +66,17 @@ const InfoModal = ({
onSuccess && onSuccess();
onClose();
toast({
title: '更新成功',
title: t('common.Update Success'),
status: 'success'
});
},
errorToast: '更新失败'
errorToast: t('common.Update Failed')
});
// 提交保存表单失败
const saveSubmitError = useCallback(() => {
// deep search message
const deepSearch = (obj: any): string => {
if (!obj) return '提交表单错误';
if (!obj) return t('common.Submit failed');
if (!!obj.message) {
return obj.message;
}
@@ -89,7 +88,7 @@ const InfoModal = ({
duration: 4000,
isClosable: true
});
}, [errors, toast]);
}, [errors, t, toast]);
const saveUpdateModel = useCallback(
() => handleSubmit((data) => saveSubmitSuccess(data), saveSubmitError)(),
@@ -111,12 +110,12 @@ const InfoModal = ({
setRefresh((state) => !state);
} catch (err: any) {
toast({
title: getErrText(err, '头像选择异常'),
title: getErrText(err, t('common.error.Select avatar failed')),
status: 'warning'
});
}
},
[setValue, toast]
[setValue, t, toast]
);
return (
@@ -127,7 +126,7 @@ const InfoModal = ({
title={t('core.app.setting')}
>
<ModalBody>
<Box>头像 & 名称</Box>
<Box>{t('core.app.Name and avatar')}</Box>
<Flex mt={2} alignItems={'center'}>
<Avatar
src={getValues('avatar')}
@@ -136,21 +135,21 @@ const InfoModal = ({
cursor={'pointer'}
borderRadius={'md'}
mr={4}
title={'点击切换头像'}
title={t('common.Set Avatar')}
onClick={() => onOpenSelectFile()}
/>
<FormControl>
<Input
bg={'myWhite.600'}
placeholder={'给应用设置一个名称'}
placeholder={t('core.app.Set a name for your app')}
{...register('name', {
required: '展示名称不能为空'
required: true
})}
></Input>
</FormControl>
</Flex>
<Box mt={4} mb={1}>
应用介绍
{t('core.app.App intro')}
</Box>
{/* <Box color={'myGray.500'} mb={2} fontSize={'sm'}>
该介绍主要用于记忆和在应用市场展示
@@ -158,7 +157,7 @@ const InfoModal = ({
<Textarea
rows={4}
maxLength={500}
placeholder={'给你的 AI 应用一个介绍'}
placeholder={t('core.app.Make a brief introduction of your app')}
bg={'myWhite.600'}
{...register('intro')}
/>
@@ -176,10 +175,10 @@ const InfoModal = ({
<ModalFooter>
<Button variant={'whiteBase'} mr={3} onClick={onClose}>
取消
{t('common.Close')}
</Button>
<Button isLoading={btnLoading} onClick={saveUpdateModel}>
保存
{t('common.Save')}
</Button>
</ModalFooter>

46
projects/app/src/pages/app/detail/components/OutLink/Share.tsx
View File

@@ -67,11 +67,8 @@ const Share = ({ appId }: { appId: string }) => {
<Box position={'relative'} pt={3} px={5} minH={'50vh'}>
<Flex justifyContent={'space-between'}>
<Box fontWeight={'bold'} fontSize={['md', 'xl']}>
免登录窗口
<MyTooltip
forceShow
label="可以直接分享该模型给其他用户去进行对话，对方无需登录即可直接进行对话。注意，这个功能会消耗你账号的余额，请保管好链接！"
>
{t('core.app.Share link')}
<MyTooltip forceShow label={t('core.app.Share link desc detail')}>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
@@ -82,29 +79,29 @@ const Share = ({ appId }: { appId: string }) => {
{...(shareChatList.length >= 10
? {
isDisabled: true,
title: '最多创建10组'
title: t('core.app.share.Amount limit tip')
}
: {})}
onClick={() => setEditLinkData(defaultOutLinkForm)}
>
创建新链接
{t('core.app.share.Create link')}
</Button>
</Flex>
<TableContainer mt={3}>
<Table variant={'simple'} w={'100%'} overflowX={'auto'} fontSize={'sm'}>
<Thead>
<Tr>
<Th>名称</Th>
<Th>金额消耗</Th>
<Th>返回引用</Th>
<Th>{t('common.Name')}</Th>
<Th>{t('common.Price used')}</Th>
<Th>{t('core.app.share.Is response quote')}</Th>
{feConfigs?.isPlus && (
<>
<Th>IP限流（人/分钟）</Th>
<Th>过期时间</Th>
<Th>身份校验</Th>
<Th>{t('core.app.share.Ip limit title')}</Th>
<Th>{t('common.Expired Time')}</Th>
<Th>{t('core.app.share.Role check')}</Th>
</>
)}
<Th>最后使用时间</Th>
<Th>{t('common.Last use time')}</Th>
<Th></Th>
</Tr>
</Thead>
@@ -117,8 +114,8 @@ const Share = ({ appId }: { appId: string }) => {
{feConfigs?.isPlus
? `${
item.limit && item.limit.credit > -1
? ` / ${item.limit.credit}元`
: ' / 无限制'
? ` / ￥${item.limit.credit}`
: ` / ${t('common.Unlimited')}`
}`
: ''}
</Td>
@@ -134,7 +131,9 @@ const Share = ({ appId }: { appId: string }) => {
<Th>{item?.limit?.hookUrl ? '✔' : '✖'}</Th>
</>
)}
<Td>{item.lastTime ? formatTimeToChatTime(item.lastTime) : '未使用'}</Td>
<Td>
{item.lastTime ? t(formatTimeToChatTime(item.lastTime)) : t('common.Un used')}
</Td>
<Td display={'flex'} alignItems={'center'}>
<Menu autoSelect={false} isLazy>
<MenuButton
@@ -197,7 +196,7 @@ const Share = ({ appId }: { appId: string }) => {
<Flex h={'100%'} flexDirection={'column'} alignItems={'center'} pt={'10vh'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
没有创建分享链接
{t('core.app.share.Not share link')}
</Box>
</Flex>
)}
@@ -208,7 +207,7 @@ const Share = ({ appId }: { appId: string }) => {
defaultData={editLinkData}
onCreate={(id) => {
const url = `${location.origin}/chat/share?shareId=${id}`;
copyData(url, '创建成功。已复制分享地址，可直接分享使用');
copyData(url, t('core.app.share.Create link tip'));
refetchShareChatList();
setEditLinkData(undefined);
}}
@@ -268,14 +267,14 @@ function EditLinkModal({
appId,
type
}),
errorToast: '创建链接异常',
errorToast: t('common.Create Failed'),
onSuccess: onCreate
});
const { mutate: onclickUpdate, isLoading: updating } = useRequest({
mutationFn: (e: OutLinkEditType) => {
return putShareChat(e);
},
errorToast: '更新链接异常',
errorToast: t('common.Update Failed'),
onSuccess: onEdit
});
@@ -384,14 +383,13 @@ function EditLinkModal({
<ModalFooter>
<Button variant={'whiteBase'} mr={3} onClick={onClose}>
取消
{t('common.Close')}
</Button>
<Button
isLoading={creating || updating}
onClick={submitShareChat((data) => (isEdit ? onclickUpdate(data) : onclickCreate(data)))}
>
确认
{t('common.Confirm')}
</Button>
</ModalFooter>
</MyModal>

12
projects/app/src/pages/app/detail/components/OutLink/index.tsx
View File

@@ -6,9 +6,11 @@ import dynamic from 'next/dynamic';
import MyRadio from '@/components/common/MyRadio';
import Share from './Share';
import { useTranslation } from 'next-i18next';
const API = dynamic(() => import('./API'));
const OutLink = ({ appId }: { appId: string }) => {
const { t } = useTranslation();
const theme = useTheme();
const [linkType, setLinkType] = useState<`${OutLinkTypeEnum}`>(OutLinkTypeEnum.share);
@@ -16,7 +18,7 @@ const OutLink = ({ appId }: { appId: string }) => {
return (
<Box pt={[1, 5]}>
<Box fontWeight={'bold'} fontSize={['md', 'xl']} mb={2} px={[4, 8]}>
外部使用途径
{t('core.app.External using')}
</Box>
<Box pb={[5, 7]} px={[4, 8]} borderBottom={theme.borders.base}>
<MyRadio
@@ -25,14 +27,14 @@ const OutLink = ({ appId }: { appId: string }) => {
list={[
{
icon: '/imgs/modal/shareFill.svg',
title: '免登录窗口',
desc: '分享链接给其他用户，无需登录即可直接进行使用',
title: t('core.app.Share link'),
desc: t('core.app.Share link desc'),
value: OutLinkTypeEnum.share
},
{
icon: 'support/outlink/apikeyFill',
title: 'API 访问',
desc: '通过 API 接入到已有系统中，或企微、飞书等',
title: t('core.app.Api request'),
desc: t('core.app.Api request desc'),
value: OutLinkTypeEnum.apikey
}
// {

8
projects/app/src/pages/app/detail/components/SimpleEdit/AppCard.tsx
View File

@@ -96,7 +96,7 @@ const AppCard = ({ appId }: { appId: string }) => {
wordBreak={'break-all'}
color={'myGray.600'}
>
{appDetail.intro || '快来给应用一个介绍~'}
{appDetail.intro || t('core.app.tip.Add a intro to app')}
</Box>
<Flex>
<Button
@@ -105,7 +105,7 @@ const AppCard = ({ appId }: { appId: string }) => {
leftIcon={<MyIcon name={'core/chat/chatLight'} w={'16px'} />}
onClick={() => router.push(`/chat?appId=${appId}`)}
>
对话
{t('core.Chat')}
</Button>
<Button
mx={3}
@@ -121,7 +121,7 @@ const AppCard = ({ appId }: { appId: string }) => {
});
}}
>
外接
{t('core.app.navbar.External')}
</Button>
{appDetail.isOwner && (
<Button
@@ -130,7 +130,7 @@ const AppCard = ({ appId }: { appId: string }) => {
leftIcon={<MyIcon name={'common/settingLight'} w={'16px'} />}
onClick={() => setSettingAppInfo(appDetail)}
>
设置
{t('common.Setting')}
</Button>
)}
</Flex>

31
projects/app/src/pages/app/detail/components/SimpleEdit/CfrEditModal.tsx
View File

@@ -1,9 +1,9 @@
import React, { useMemo, useState } from 'react';
import React, { useCallback, useState, useTransition } from 'react';
import MyModal from '@/components/MyModal';
import { useTranslation } from 'next-i18next';
import { Button, ModalBody, ModalFooter } from '@chakra-ui/react';
import PromptTextarea from '@/components/common/Textarea/PromptTextarea';
import { Box, Button, ModalBody, ModalFooter } from '@chakra-ui/react';
import PromptEditor from '@fastgpt/web/components/common/Textarea/PromptEditor';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
@@ -18,6 +18,7 @@ const CfrEditModal = ({
}) => {
const { t } = useTranslation();
const [value, setValue] = useState(defaultValue);
const [, startTst] = useTransition();
return (
<MyModal
@@ -32,17 +33,19 @@ const CfrEditModal = ({
<MyTooltip label={t('core.app.edit.cfr background tip')} forceShow>
<QuestionOutlineIcon display={['none', 'inline']} ml={1} />
</MyTooltip>
<PromptTextarea
mt={1}
flex={1}
bg={'myWhite.400'}
rows={5}
placeholder={t('core.module.input.placeholder.cfr background')}
defaultValue={value}
onBlur={(e) => {
setValue(e.target.value || '');
}}
/>
<Box mt={1} flex={1}>
<PromptEditor
h={200}
showOpenModal={false}
placeholder={t('core.module.input.placeholder.cfr background')}
defaultValue={value}
onChange={useCallback((e: string) => {
startTst(() => {
setValue(e);
});
}, [])}
/>
</Box>
</ModalBody>
<ModalFooter>
<Button

2
projects/app/src/pages/app/detail/components/SimpleEdit/ChatTest.tsx
View File

@@ -114,7 +114,7 @@ const ChatTest = ({ appId }: { appId: string }) => {
right={0}
left={0}
bottom={0}
bg={'rgba(255,255,255,0.6)'}
bg={'rgba(255,255,255,0.7)'}
alignItems={'center'}
justifyContent={'center'}
flexDirection={'column'}

118
projects/app/src/pages/app/detail/components/SimpleEdit/EditForm.tsx
View File

@@ -1,4 +1,4 @@
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import React, { useMemo, useState, useTransition } from 'react';
import {
Box,
Flex,
@@ -24,7 +24,6 @@ import { useTranslation } from 'next-i18next';
import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useAppStore } from '@/web/core/app/store/useAppStore';
import { postForm2Modules } from '@/web/core/app/utils';
import dynamic from 'next/dynamic';
@@ -34,9 +33,11 @@ import Avatar from '@/components/Avatar';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { SimpleModeTemplate_FastGPT_Universal } from '@/global/core/app/constants';
import VariableEdit from '@/components/core/module/Flow/components/modules/VariableEdit';
import PromptTextarea from '@/components/common/Textarea/PromptTextarea/index';
import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constant';
import MyTextarea from '@/components/common/Textarea/MyTextarea/index';
import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constants';
import SelectAiModel from '@/components/Select/SelectAiModel';
import PromptEditor from '@fastgpt/web/components/common/Textarea/PromptEditor';
import { formatVariablesIcon } from '@fastgpt/global/core/module/utils';
const DatasetSelectModal = dynamic(() => import('@/components/core/module/DatasetSelectModal'));
const DatasetParamsModal = dynamic(() => import('@/components/core/module/DatasetParamsModal'));
@@ -61,8 +62,9 @@ const EditForm = ({
const { loadAllDatasets, allDatasets } = useDatasetStore();
const { isPc } = useSystemStore();
const [refresh, setRefresh] = useState(false);
const [, startTst] = useTransition();
const { register, setValue, getValues, reset, handleSubmit, control } =
const { setValue, getValues, reset, handleSubmit, control, watch } =
useForm<AppSimpleEditFormType>({
defaultValues: getDefaultAppForm()
});
@@ -97,24 +99,25 @@ const EditForm = ({
content: t('core.app.edit.Confirm Save App Tip')
});
const chatModelSelectList = useMemo(() => {
return chatModelList.map((item) => ({
const variables = watch('userGuide.variables');
const formatVariables = useMemo(() => formatVariablesIcon(variables), [variables]);
const aiSystemPrompt = watch('aiSettings.systemPrompt');
const searchMode = watch('dataset.searchMode');
const chatModelSelectList = (() =>
chatModelList.map((item) => ({
value: item.model,
label: item.name
}));
}, [refresh]);
})))();
const selectDatasets = useMemo(
() => allDatasets.filter((item) => datasets.find((dataset) => dataset.datasetId === item._id)),
[allDatasets, datasets]
);
const selectSimpleTemplate = useMemo(
() =>
simpleModeTemplates?.find((item) => item.id === getValues('templateId')) ||
SimpleModeTemplate_FastGPT_Universal,
[getValues, refresh]
);
const selectSimpleTemplate = (() =>
simpleModeTemplates?.find((item) => item.id === getValues('templateId')) ||
SimpleModeTemplate_FastGPT_Universal)();
const tokenLimit = useMemo(() => {
return (
@@ -124,10 +127,9 @@ const EditForm = ({
}, [getValues, refresh]);
const datasetSearchMode = useMemo(() => {
const mode = getValues('dataset.searchMode');
if (!mode) return '';
return t(DatasetSearchModeMap[mode]?.title);
}, [getValues, t, refresh]);
if (!searchMode) return '';
return t(DatasetSearchModeMap[searchMode]?.title);
}, [searchMode, t]);
const { mutate: onSubmitSave, isLoading: isSaving } = useRequest({
mutationFn: async (data: AppSimpleEditFormType) => {
@@ -144,21 +146,21 @@ const EditForm = ({
errorToast: t('common.Save Failed')
});
const appModule2Form = useCallback(() => {
const formVal = appModules2Form({
templateId: appDetail.simpleTemplateId,
modules: appDetail.modules
});
reset(formVal);
setTimeout(() => {
setRefresh((state) => !state);
}, 100);
}, [appDetail.modules, appDetail.simpleTemplateId, reset]);
useEffect(() => {
appModule2Form();
}, [appModule2Form]);
const { isSuccess: isInitd } = useQuery(
['init', appDetail],
() => {
const formatVal = appModules2Form({
templateId: appDetail.simpleTemplateId,
modules: appDetail.modules
});
reset(formatVal);
setRefresh(!refresh);
return formatVal;
},
{
enabled: !!appDetail._id
}
);
useQuery(['loadAllDatasets'], loadAllDatasets);
const BoxStyles: BoxProps = {
@@ -229,15 +231,15 @@ const EditForm = ({
{/* simple mode select */}
<Flex {...BoxStyles}>
<Flex alignItems={'center'} flex={'1 0 0'}>
<Image alt={''} src={'/imgs/module/templates.png'} w={'18px'} />
<MyIcon name={'core/app/simpleMode/template'} w={'20px'} />
<Box mx={2}>{t('core.app.simple.mode template select')}</Box>
</Flex>
<MySelect
w={['200px', '250px']}
list={
simpleModeTemplates?.map((item) => ({
alias: item.name,
label: item.desc,
alias: t(item.name),
label: t(item.desc),
value: item.id
})) || []
}
@@ -253,7 +255,7 @@ const EditForm = ({
{selectSimpleTemplate?.systemForm?.aiSettings && (
<Box {...BoxStyles}>
<Flex alignItems={'center'}>
<Image alt={''} src={'/imgs/module/AI.png'} w={'18px'} />
<MyIcon name={'core/app/simpleMode/ai'} w={'20px'} />
<Box ml={2} flex={1}>
{t('app.AI Settings')}
</Box>
@@ -263,7 +265,7 @@ const EditForm = ({
selectSimpleTemplate.systemForm.aiSettings.quotePrompt) && (
<Flex {...BoxBtnStyles} onClick={onOpenAIChatSetting}>
<MyIcon mr={1} name={'common/settingLight'} w={'14px'} />
{t('app.Open AI Advanced Settings')}
{t('common.More settings')}
</Flex>
)}
</Flex>
@@ -293,20 +295,23 @@ const EditForm = ({
<Flex mt={10} alignItems={'flex-start'}>
<Box {...LabelStyles}>
{t('core.ai.Prompt')}
<MyTooltip label={chatNodeSystemPromptTip} forceShow>
<MyTooltip label={t(chatNodeSystemPromptTip)} forceShow>
<QuestionOutlineIcon display={['none', 'inline']} ml={1} />
</MyTooltip>
</Box>
<PromptTextarea
flex={1}
bg={'myWhite.400'}
rows={5}
placeholder={chatNodeSystemPromptTip}
defaultValue={getValues('aiSettings.systemPrompt')}
onBlur={(e) => {
setValue('aiSettings.systemPrompt', e.target.value || '');
}}
/>
{isInitd && (
<PromptEditor
defaultValue={aiSystemPrompt}
onChange={(text) => {
startTst(() => {
setValue('aiSettings.systemPrompt', text);
});
}}
variables={formatVariables}
placeholder={t('core.app.tip.chatNodeSystemPromptTip')}
title={t('core.ai.Prompt')}
/>
)}
</Flex>
)}
</Box>
@@ -317,7 +322,7 @@ const EditForm = ({
<Box {...BoxStyles}>
<Flex alignItems={'center'}>
<Flex alignItems={'center'} flex={1}>
<Image alt={''} src={'/imgs/module/db.png'} w={'18px'} />
<MyIcon name={'core/app/simpleMode/dataset'} w={'20px'} />
<Box ml={2}>{t('core.dataset.Choose Dataset')}</Box>
</Flex>
{selectSimpleTemplate.systemForm.dataset.datasets && (
@@ -409,7 +414,7 @@ const EditForm = ({
{selectSimpleTemplate?.systemForm?.userGuide?.variables && (
<Box {...BoxStyles}>
<VariableEdit
variables={getValues('userGuide.variables')}
variables={variables}
onChange={(e) => {
setValue('userGuide.variables', e);
setRefresh(!refresh);
@@ -422,17 +427,17 @@ const EditForm = ({
{selectSimpleTemplate?.systemForm?.userGuide?.welcomeText && (
<Box {...BoxStyles}>
<Flex alignItems={'center'}>
<Image alt={''} src={'/imgs/module/userGuide.png'} w={'18px'} />
<MyIcon name={'core/app/simpleMode/chat'} w={'20px'} />
<Box mx={2}>{t('core.app.Welcome Text')}</Box>
<MyTooltip label={welcomeTextTip} forceShow>
<MyTooltip label={t(welcomeTextTip)} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Flex>
<PromptTextarea
<MyTextarea
mt={2}
bg={'myWhite.400'}
rows={5}
placeholder={welcomeTextTip}
placeholder={t(welcomeTextTip)}
defaultValue={getValues('userGuide.welcomeText')}
onBlur={(e) => {
setValue('userGuide.welcomeText', e.target.value || '');
@@ -481,6 +486,7 @@ const EditForm = ({
}}
defaultData={getValues('aiSettings')}
simpleModeTemplate={selectSimpleTemplate}
pickerMenu={formatVariables}
/>
)}
{isOpenDatasetSelect && (

38
projects/app/src/pages/app/detail/index.tsx
View File

@@ -16,6 +16,7 @@ import SimpleEdit from './components/SimpleEdit';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { useAppStore } from '@/web/core/app/store/useAppStore';
import Head from 'next/head';
import { useTranslation } from 'next-i18next';
const FlowEdit = dynamic(() => import('./components/FlowEdit'), {
loading: () => <Loading />
@@ -32,6 +33,7 @@ enum TabEnum {
}
const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
const { t } = useTranslation();
const router = useRouter();
const theme = useTheme();
const { toast } = useToast();
@@ -52,23 +54,39 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
const tabList = useMemo(
() => [
{ label: '简易配置', id: TabEnum.simpleEdit, icon: 'common/overviewLight' },
{
label: t('core.app.navbar.Simple mode'),
id: TabEnum.simpleEdit,
icon: 'common/overviewLight'
},
...(feConfigs?.hide_app_flow
? []
: [{ label: '高级编排', id: TabEnum.adEdit, icon: 'common/settingLight' }]),
{ label: '外部使用', id: TabEnum.outLink, icon: 'support/outlink/shareLight' },
{ label: '对话日志', id: TabEnum.logs, icon: 'core/app/logsLight' },
{ label: '立即对话', id: TabEnum.startChat, icon: 'core/chat/chatLight' }
: [
{
label: t('core.app.navbar.Flow mode'),
id: TabEnum.adEdit,
icon: 'core/modules/flowLight'
}
]),
{
label: t('core.app.navbar.External'),
id: TabEnum.outLink,
icon: 'support/outlink/shareLight'
},
{ label: t('app.Chat logs'), id: TabEnum.logs, icon: 'core/app/logsLight' },
{ label: t('core.Start chat'), id: TabEnum.startChat, icon: 'core/chat/chatLight' }
],
[]
[t]
);
const onCloseFlowEdit = useCallback(() => setCurrentTab(TabEnum.simpleEdit), [setCurrentTab]);
useEffect(() => {
const listen =
process.env.NODE_ENV === 'production'
? (e: any) => {
e.preventDefault();
e.returnValue = '内容已修改，确认离开页面吗？';
e.returnValue = t('core.common.tip.leave page');
}
: () => {};
window.addEventListener('beforeunload', listen);
@@ -82,7 +100,7 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
useQuery([appId], () => loadAppDetail(appId, true), {
onError(err: any) {
toast({
title: err?.message || '获取应用异常',
title: err?.message || t('core.app.error.Get app failed'),
status: 'error'
});
router.replace('/app/list');
@@ -146,7 +164,7 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
borderRadius={'50%'}
aria-label={''}
/>
我的应用
{t('app.My Apps')}
</Flex>
</Box>
{/* phone tab */}
@@ -173,7 +191,7 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
<Box flex={'1 0 0'} h={[0, '100%']} overflow={['overlay', '']}>
{currentTab === TabEnum.simpleEdit && <SimpleEdit appId={appId} />}
{currentTab === TabEnum.adEdit && appDetail && (
<FlowEdit app={appDetail} onClose={() => setCurrentTab(TabEnum.simpleEdit)} />
<FlowEdit app={appDetail} onClose={onCloseFlowEdit} />
)}
{currentTab === TabEnum.logs && <Logs appId={appId} />}
{currentTab === TabEnum.outLink && <OutLink appId={appId} />}

27
projects/app/src/pages/app/list/component/CreateModal.tsx
View File

@@ -3,7 +3,6 @@ import {
Box,
Flex,
Button,
ModalHeader,
ModalFooter,
ModalBody,
Input,
@@ -69,19 +68,19 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
setRefresh((state) => !state);
} catch (err: any) {
toast({
title: getErrText(err, '头像选择异常'),
title: getErrText(err, t('common.error.Select avatar failed')),
status: 'warning'
});
}
},
[setValue, toast]
[setValue, t, toast]
);
const { mutate: onclickCreate, isLoading: creating } = useRequest({
mutationFn: async (data: FormType) => {
const template = appTemplates.find((item) => item.id === data.templateId);
if (!template) {
return Promise.reject('模板不存在');
return Promise.reject(t('core.dataset.error.Template does not exist'));
}
return postCreateApp({
avatar: data.avatar,
@@ -95,8 +94,8 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
onSuccess();
onClose();
},
successToast: '创建成功',
errorToast: '创建应用异常'
successToast: t('common.Create Success'),
errorToast: t('common.Create Failed')
});
return (
@@ -109,10 +108,10 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
>
<ModalBody>
<Box color={'myGray.800'} fontWeight={'bold'}>
取个响亮的名字
{t('common.Set Name')}
</Box>
<Flex mt={3} alignItems={'center'}>
<MyTooltip label={'点击设置头像'}>
<MyTooltip label={t('common.Set Avatar')}>
<Avatar
flexShrink={0}
src={getValues('avatar')}
@@ -129,14 +128,14 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
autoFocus
bg={'myWhite.600'}
{...register('name', {
required: '应用名不能为空~'
required: t('core.app.error.App name can not be empty')
})}
/>
</Flex>
{!feConfigs?.hide_app_flow && (
<>
<Box mt={[4, 7]} mb={[0, 3]} color={'myGray.800'} fontWeight={'bold'}>
从模板中选择
{t('core.app.Select app from template')}
</Box>
<Grid
userSelect={'none'}
@@ -168,11 +167,11 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
<Flex alignItems={'center'}>
<Avatar src={item.avatar} borderRadius={'md'} w={'20px'} />
<Box ml={3} fontWeight={'bold'}>
{item.name}
{t(item.name)}
</Box>
</Flex>
<Box fontSize={'sm'} mt={4}>
{item.intro}
{t(item.intro)}
</Box>
</Card>
))}
@@ -183,10 +182,10 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: (
<ModalFooter>
<Button variant={'whiteBase'} mr={3} onClick={onClose}>
取消
{t('common.Close')}
</Button>
<Button isLoading={creating} onClick={handleSubmit((data) => onclickCreate(data))}>
确认创建
{t('common.Confirm Create')}
</Button>
</ModalFooter>

1
projects/app/src/pages/chat/index.tsx
View File

@@ -348,7 +348,6 @@ const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => {
userGuideModule={chatData.app?.userGuideModule}
showFileSelector={checkChatSupportSelectFileByChatModels(chatData.app.chatModels)}
feedbackType={'user'}
onUpdateVariable={(e) => {}}
onStartChat={startChat}
onDelMessage={(e) => delOneHistoryItem({ ...e, appId, chatId })}
appId={appId}

7
projects/app/src/pages/dataset/component/EditFolderModal.tsx
View File

@@ -42,7 +42,7 @@ const EditFolderModal = ({
});
return (
<MyModal isOpen onClose={onClose} iconSrc="/imgs/modal/folder.svg" title={typeMap.title}>
<MyModal isOpen onClose={onClose} iconSrc="common/folderFill" title={typeMap.title}>
<ModalBody>
<Input
ref={inputRef}
@@ -53,11 +53,8 @@ const EditFolderModal = ({
/>
</ModalBody>
<ModalFooter>
<Button mr={3} variant={'whiteBase'} onClick={onClose}>
{t('Cancel')}
</Button>
<Button isLoading={isLoading} onClick={onSave}>
{t('Confirm')}
{t('common.Confirm')}
</Button>
</ModalFooter>
</MyModal>

87
projects/app/src/pages/dataset/detail/components/CollectionCard.tsx
View File

@@ -41,14 +41,13 @@ import { useEditTitle } from '@/web/common/hooks/useEditTitle';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
import EmptyTip from '@/components/EmptyTip';
import {
FolderAvatarSrc,
DatasetCollectionTypeEnum,
TrainingModeEnum,
DatasetTypeEnum,
DatasetTypeMap,
DatasetStatusEnum,
DatasetCollectionSyncResultMap
} from '@fastgpt/global/core/dataset/constant';
} from '@fastgpt/global/core/dataset/constants';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import EditFolderModal, { useEditFolder } from '../../component/EditFolderModal';
import { TabEnum } from '..';
@@ -62,11 +61,12 @@ import { useUserStore } from '@/web/support/user/useUserStore';
import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import { DatasetCollectionSyncResultEnum } from '../../../../../../../packages/global/core/dataset/constant';
import { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants';
import MyBox from '@/components/common/MyBox';
import { ImportDataSourceEnum } from './Import';
const FileImportModal = dynamic(() => import('./Import/ImportModal'), {});
const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {});
const FileSourceSelector = dynamic(() => import('./Import/sourceSelector/FileSourceSelector'), {});
const CollectionCard = () => {
const BoxRef = useRef<HTMLDivElement>(null);
@@ -90,9 +90,9 @@ const CollectionCard = () => {
});
const {
isOpen: isOpenFileImportModal,
onOpen: onOpenFileImportModal,
onClose: onCloseFileImportModal
isOpen: isOpenFileSourceSelector,
onOpen: onOpenFileSourceSelector,
onClose: onCloseFileSourceSelector
} = useDisclosure();
const {
isOpen: isOpenWebsiteModal,
@@ -159,12 +159,16 @@ const CollectionCard = () => {
statusText: t('dataset.collections.Collection Embedding', {
total: collection.trainingAmount
}),
color: 'myGray.500'
color: 'myGray.600',
bg: 'myGray.50',
borderColor: 'borderColor.low'
};
}
return {
statusText: t('core.dataset.collection.status.active'),
color: 'green.500'
color: 'green.600',
bg: 'green.50',
borderColor: 'green.300'
};
})();
@@ -299,7 +303,8 @@ const CollectionCard = () => {
return (
<MyBox isLoading={isLoading} h={'100%'} py={[2, 4]}>
<Flex ref={BoxRef} flexDirection={'column'} py={[1, 3]} h={'100%'}>
<Flex px={[2, 6]} alignItems={['flex-start', 'center']} h={'35px'}>
{/* header */}
<Flex px={[2, 6]} alignItems={'flex-start'} h={'35px'}>
<Box flex={1}>
<ParentPath
paths={paths.map((path, i) => ({
@@ -343,7 +348,7 @@ const CollectionCard = () => {
<MyInput
bg={'myGray.50'}
w={['100%', '250px']}
size={['sm', 'md']}
size={'sm'}
h={'36px'}
placeholder={t('common.Search') || ''}
value={searchText}
@@ -376,7 +381,7 @@ const CollectionCard = () => {
<>
{userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
<MyMenu
offset={[-40, 10]}
offset={[-0, 10]}
width={120}
Button={
<MenuButton
@@ -405,7 +410,7 @@ const CollectionCard = () => {
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={2} />
<MyIcon name={'common/folderFill'} w={'20px'} mr={2} />
{t('Folder')}
</Flex>
),
@@ -414,7 +419,7 @@ const CollectionCard = () => {
{
child: (
<Flex>
<Image src={'/imgs/files/collection.svg'} alt={''} w={'20px'} mr={2} />
<MyIcon name={'core/dataset/manualCollection'} mr={2} w={'20px'} />
{t('core.dataset.Manual collection')}
</Flex>
),
@@ -430,11 +435,27 @@ const CollectionCard = () => {
{
child: (
<Flex>
<Image src={'/imgs/files/file.svg'} alt={''} w={'20px'} mr={2} />
{t('core.dataset.File collection')}
<MyIcon name={'core/dataset/fileCollection'} mr={2} w={'20px'} />
{t('core.dataset.Text collection')}
</Flex>
),
onClick: onOpenFileImportModal
onClick: onOpenFileSourceSelector
},
{
child: (
<Flex>
<MyIcon name={'core/dataset/tableCollection'} mr={2} w={'20px'} />
{t('core.dataset.Table collection')}
</Flex>
),
onClick: () =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.import,
source: ImportDataSourceEnum.tableLocal
}
})
}
]}
/>
@@ -478,6 +499,7 @@ const CollectionCard = () => {
)}
</Flex>
{/* collection table */}
<TableContainer
px={[2, 6]}
mt={[0, 3]}
@@ -545,11 +567,6 @@ const CollectionCard = () => {
} catch (error) {}
setDragTargetId(undefined);
}}
title={
collection.type === DatasetCollectionTypeEnum.folder
? t('dataset.collections.Click to view folder')
: t('dataset.collections.Click to view file')
}
onClick={() => {
if (collection.type === DatasetCollectionTypeEnum.folder) {
router.replace({
@@ -572,7 +589,7 @@ const CollectionCard = () => {
<Td w={'50px'}>{index + 1}</Td>
<Td minW={'150px'} maxW={['200px', '300px']} draggable>
<Flex alignItems={'center'}>
<Image src={collection.icon} w={'16px'} mr={2} alt={''} />
<MyIcon name={collection.icon as any} w={'16px'} mr={2} />
<MyTooltip label={t('common.folder.Drag Tip')} shouldWrapChildren={false}>
<Box fontWeight={'bold'} className="textEllipsis">
{collection.name}
@@ -583,19 +600,28 @@ const CollectionCard = () => {
<Td fontSize={'md'}>{collection.dataAmount || '-'}</Td>
<Td>{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')}</Td>
<Td>
<Flex
<Box
display={'inline-flex'}
alignItems={'center'}
w={'auto'}
color={collection.color}
bg={collection.bg}
borderWidth={'1px'}
borderColor={collection.borderColor}
px={3}
py={1}
borderRadius={'md'}
_before={{
content: '""',
w: '10px',
h: '10px',
w: '6px',
h: '6px',
mr: 2,
borderRadius: 'lg',
bg: collection.color
}}
>
{t(collection.statusText)}
</Flex>
</Box>
</Td>
<Td onClick={(e) => e.stopPropagation()}>
{collection.canWrite && userInfo?.team?.role !== TeamMemberRoleEnum.visitor && (
@@ -744,8 +770,8 @@ const CollectionCard = () => {
<ConfirmDeleteModal />
<ConfirmSyncModal />
<EditTitleModal />
<EditCreateVirtualFileModal />
{isOpenFileImportModal && (
<EditCreateVirtualFileModal iconSrc={'modal/manualDataset'} closeBtnText={''} />
{/* {isOpenFileImportModal && (
<FileImportModal
datasetId={datasetId}
parentId={parentId}
@@ -755,7 +781,8 @@ const CollectionCard = () => {
}}
onClose={onCloseFileImportModal}
/>
)}
)} */}
{isOpenFileSourceSelector && <FileSourceSelector onClose={onCloseFileSourceSelector} />}
{!!editFolderData && (
<EditFolderModal
onClose={() => setEditFolderData(undefined)}

415
projects/app/src/pages/dataset/detail/components/DataCard.tsx
View File

@@ -43,7 +43,7 @@ import {
DatasetCollectionTypeMap,
TrainingModeEnum,
TrainingTypeMap
} from '@fastgpt/global/core/dataset/constant';
} from '@fastgpt/global/core/dataset/constants';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { getFileAndOpen } from '@/web/core/dataset/utils';
@@ -65,7 +65,8 @@ const DataCard = () => {
const [searchText, setSearchText] = useState('');
const { toast } = useToast();
const { openConfirm, ConfirmModal } = useConfirm({
content: t('dataset.Confirm to delete the data')
content: t('dataset.Confirm to delete the data'),
type: 'delete'
});
const { isOpen, onOpen, onClose } = useDisclosure();
@@ -173,202 +174,224 @@ const DataCard = () => {
}, [collection, t]);
return (
<Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
<Flex alignItems={'center'}>
<IconButton
mr={3}
icon={<MyIcon name={'common/backFill'} w={['14px', '18px']} color={'primary.500'} />}
variant={'whitePrimary'}
size={'smSquare'}
borderRadius={'50%'}
aria-label={''}
onClick={() =>
router.replace({
query: {
datasetId: router.query.datasetId,
parentId: router.query.parentId,
currentTab: TabEnum.collectionCard
}
})
}
/>
<Flex className="textEllipsis" flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
<Box lineHeight={1.2}>
<RawSourceBox
sourceName={collection?.name}
sourceId={collection?.fileId || collection?.rawLink}
fontSize={['md', 'lg']}
color={'black'}
textDecoration={'none'}
/>
<Box fontSize={'sm'} color={'myGray.500'}>
{t('core.dataset.collection.id')}:{' '}
<Box as={'span'} userSelect={'all'}>
{collection?._id}
<Box position={'relative'} py={[1, 5]} h={'100%'}>
<Flex ref={BoxRef} flexDirection={'column'} h={'100%'}>
<Flex alignItems={'center'} px={5}>
<IconButton
mr={3}
icon={<MyIcon name={'common/backFill'} w={['14px', '18px']} color={'primary.500'} />}
variant={'whitePrimary'}
size={'smSquare'}
borderRadius={'50%'}
aria-label={''}
onClick={() =>
router.replace({
query: {
datasetId: router.query.datasetId,
parentId: router.query.parentId,
currentTab: TabEnum.collectionCard
}
})
}
/>
<Flex className="textEllipsis" flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
<Box lineHeight={1.2}>
<RawSourceBox
sourceName={collection?.name}
sourceId={collection?.fileId || collection?.rawLink}
fontSize={['md', 'lg']}
color={'black'}
textDecoration={'none'}
/>
<Box fontSize={'sm'} color={'myGray.500'}>
{t('core.dataset.collection.id')}:{' '}
<Box as={'span'} userSelect={'all'}>
{collection?._id}
</Box>
</Box>
</Box>
</Box>
</Flex>
{canWrite && (
<Box>
<Button
mx={2}
variant={'whitePrimary'}
size={['sm', 'md']}
onClick={() => {
if (!collection) return;
setEditDataId('');
}}
>
{t('dataset.Insert Data')}
</Button>
</Box>
)}
{isPc && (
<MyTooltip label={t('core.dataset.collection.metadata.Read Metadata')}>
<IconButton
variant={'whiteBase'}
size={['sm', 'md']}
icon={<MyIcon name={'menu'} w={'18px'} />}
aria-label={''}
onClick={onOpen}
/>
</MyTooltip>
)}
</Flex>
{canWrite && (
<Flex my={3} alignItems={'center'} px={5}>
<Box>
<Button
mx={2}
variant={'whitePrimary'}
size={['sm', 'md']}
onClick={() => {
if (!collection) return;
setEditDataId('');
}}
>
{t('dataset.Insert Data')}
</Button>
<Box as={'span'} fontSize={['md', 'lg']}>
{t('core.dataset.data.Total Amount', { total })}
</Box>
</Box>
)}
{isPc && (
<MyTooltip label={t('core.dataset.collection.metadata.Read Metadata')}>
<IconButton
variant={'whiteBase'}
size={['sm', 'md']}
icon={<MyIcon name={'menu'} w={'18px'} />}
aria-label={''}
onClick={onOpen}
/>
</MyTooltip>
)}
</Flex>
<Flex my={3} alignItems={'center'}>
<Box>
<Box as={'span'} fontSize={['md', 'lg']}>
{t('core.dataset.data.Total Amount', { total })}
</Box>
</Box>
<Box flex={1} mr={1} />
<MyInput
leftIcon={
<MyIcon
name="common/searchLight"
position={'absolute'}
w={'14px'}
color={'myGray.500'}
/>
}
w={['200px', '300px']}
placeholder={t('core.dataset.data.Search data placeholder')}
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
getFirstData();
}}
onBlur={() => {
if (searchText === lastSearch.current) return;
getFirstData();
}}
onKeyDown={(e) => {
if (searchText === lastSearch.current) return;
if (e.key === 'Enter') {
getFirstData();
<Box flex={1} mr={1} />
<MyInput
leftIcon={
<MyIcon
name="common/searchLight"
position={'absolute'}
w={'14px'}
color={'myGray.500'}
/>
}
}}
/>
</Flex>
<Grid
minH={'100px'}
gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)', 'repeat(4,1fr)']}
gridGap={4}
>
{datasetDataList.map((item, index) => (
<Card
key={item._id}
cursor={'pointer'}
p={3}
userSelect={'none'}
boxShadow={'none'}
bg={'myWhite.500'}
border={theme.borders.sm}
position={'relative'}
overflow={'hidden'}
_hover={{
borderColor: 'myGray.200',
boxShadow: 'lg',
bg: 'white',
'& .footer': { h: 'auto', p: 3 }
w={['200px', '300px']}
placeholder={t('core.dataset.data.Search data placeholder')}
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
getFirstData();
}}
onClick={() => {
if (!collection) return;
setEditDataId(item._id);
onBlur={() => {
if (searchText === lastSearch.current) return;
getFirstData();
}}
onKeyDown={(e) => {
if (searchText === lastSearch.current) return;
if (e.key === 'Enter') {
getFirstData();
}
}}
/>
</Flex>
<Box flex={'1 0 0'} overflow={'auto'} px={5}>
<Grid
gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)', 'repeat(4,1fr)']}
gridGap={4}
>
<Flex zIndex={1} alignItems={'center'} justifyContent={'space-between'}>
<Box border={theme.borders.base} px={2} fontSize={'sm'} mr={1} borderRadius={'md'}>
# {item.chunkIndex ?? '-'}
</Box>
<Box className={'textEllipsis'} color={'myGray.500'} fontSize={'xs'}>
ID:{item._id}
{datasetDataList.map((item, index) => (
<Card
key={item._id}
cursor={'pointer'}
p={3}
userSelect={'none'}
boxShadow={'none'}
bg={'myWhite.500'}
border={theme.borders.sm}
position={'relative'}
overflow={'hidden'}
_hover={{
borderColor: 'myGray.200',
boxShadow: 'lg',
bg: 'white',
'& .footer': { h: 'auto', p: 3 }
}}
onClick={() => {
if (!collection) return;
setEditDataId(item._id);
}}
>
<Flex zIndex={1} alignItems={'center'} justifyContent={'space-between'}>
<Box
border={theme.borders.base}
px={2}
fontSize={'sm'}
mr={1}
borderRadius={'md'}
>
# {item.chunkIndex ?? '-'}
</Box>
<Box className={'textEllipsis'} color={'myGray.500'} fontSize={'xs'}>
ID:{item._id}
</Box>
</Flex>
<Box
maxH={'135px'}
minH={'90px'}
overflow={'hidden'}
wordBreak={'break-all'}
pt={1}
pb={3}
fontSize={'13px'}
>
<Box color={'black'} mb={1}>
{item.q}
</Box>
<Box color={'myGray.700'}>{item.a}</Box>
<Flex
className="footer"
position={'absolute'}
top={0}
bottom={0}
left={0}
right={0}
h={'0'}
overflow={'hidden'}
p={0}
bg={'linear-gradient(to top, white,white 20%, rgba(255,255,255,0) 60%)'}
alignItems={'flex-end'}
fontSize={'sm'}
>
<Flex alignItems={'center'}>
<MyIcon name="common/text/t" w={'14px'} mr={1} color={'myGray.500'} />
{item.q.length + (item.a?.length || 0)}
</Flex>
<Box flex={1} />
{canWrite && (
<IconButton
display={'flex'}
icon={<DeleteIcon />}
variant={'whiteDanger'}
size={'xsSquare'}
aria-label={'delete'}
onClick={(e) => {
e.stopPropagation();
openConfirm(async () => {
try {
setIsLoading(true);
await delOneDatasetDataById(item._id);
getData(pageNum);
} catch (error) {
toast({
title: getErrText(error),
status: 'error'
});
}
setIsLoading(false);
})();
}}
/>
)}
</Flex>
</Box>
</Card>
))}
</Grid>
{total > pageSize && (
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
)}
{total === 0 && (
<Flex flexDirection={'column'} alignItems={'center'} pt={'10vh'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
{t('core.dataset.data.Empty Tip')}
</Box>
</Flex>
<Box
maxH={'135px'}
minH={'90px'}
overflow={'hidden'}
wordBreak={'break-all'}
pt={1}
pb={3}
fontSize={'13px'}
>
<Box color={'black'} mb={1}>
{item.q}
</Box>
<Box color={'myGray.700'}>{item.a}</Box>
<Flex
className="footer"
position={'absolute'}
top={0}
bottom={0}
left={0}
right={0}
h={'0'}
overflow={'hidden'}
p={0}
bg={'linear-gradient(to top, white,white 20%, rgba(255,255,255,0) 60%)'}
alignItems={'flex-end'}
fontSize={'sm'}
>
<Flex alignItems={'center'}>
<MyIcon name="common/text/t" w={'14px'} mr={1} color={'myGray.500'} />
{item.q.length + (item.a?.length || 0)}
</Flex>
<Box flex={1} />
{canWrite && (
<IconButton
display={'flex'}
icon={<DeleteIcon />}
variant={'whiteDanger'}
size={'xsSquare'}
aria-label={'delete'}
onClick={(e) => {
e.stopPropagation();
openConfirm(async () => {
try {
setIsLoading(true);
await delOneDatasetDataById(item._id);
getData(pageNum);
} catch (error) {
toast({
title: getErrText(error),
status: 'error'
});
}
setIsLoading(false);
})();
}}
/>
)}
</Flex>
</Box>
</Card>
))}
</Grid>
)}
</Box>
</Flex>
{/* metadata drawer */}
<Drawer isOpen={isOpen} placement="right" size={'md'} onClose={onClose}>
@@ -378,8 +401,8 @@ const DataCard = () => {
<DrawerBody>
{metadataList.map((item) => (
<Flex key={item.label} alignItems={'center'} mb={5}>
<Box color={'myGray.500'} w={'100px'}>
<Flex key={item.label} alignItems={'center'} mb={5} wordBreak={'break-all'}>
<Box color={'myGray.500'} flex={'0 0 100px'}>
{item.label}
</Box>
<Box>{item.value}</Box>
@@ -403,20 +426,6 @@ const DataCard = () => {
</DrawerContent>
</Drawer>
{total > pageSize && (
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
)}
{total === 0 && (
<Flex flexDirection={'column'} alignItems={'center'} pt={'10vh'}>
<MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
<Box mt={2} color={'myGray.500'}>
{t('core.dataset.data.Empty Tip')}
</Box>
</Flex>
)}
{editDataId !== undefined && collection && (
<InputDataModal
collectionId={collection._id}

166
projects/app/src/pages/dataset/detail/components/Import/Chunk.tsx
View File

@@ -1,166 +0,0 @@
import React from 'react';
import {
Box,
Flex,
Button,
NumberInput,
NumberInputField,
NumberInputStepper,
NumberIncrementStepper,
NumberDecrementStepper,
Input,
Grid
} from '@chakra-ui/react';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
import { useTranslation } from 'next-i18next';
const fileExtension = '.txt, .docx, .pdf, .md, .html';
const ChunkImport = () => {
const { t } = useTranslation();
const { datasetDetail } = useDatasetStore();
const vectorModel = datasetDetail.vectorModel;
const unitPrice = vectorModel?.inputPrice || 0.002;
const {
chunkLen,
setChunkLen,
setCustomSplitChar,
successChunks,
totalChunks,
totalTokens,
isUnselectedFile,
price,
onclickUpload,
onReSplitChunks,
uploading,
showRePreview,
setReShowRePreview
} = useImportStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: t('core.dataset.import.Import Tip')
});
return (
<Box display={['block', 'flex']} h={['auto', '100%']}>
<SelectorContainer fileExtension={fileExtension}>
{/* chunk size */}
<Box mt={4} alignItems={'center'}>
<Box>
{t('core.dataset.import.Ideal chunk length')}
<MyTooltip label={t('core.dataset.import.Ideal chunk length Tips')} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Box>
<Box
mt={1}
css={{
'& > span': {
display: 'block'
}
}}
>
<MyTooltip
label={t('core.dataset.import.Chunk Range', {
max: datasetDetail.vectorModel.maxToken
})}
>
<NumberInput
defaultValue={chunkLen}
min={100}
max={datasetDetail.vectorModel.maxToken}
step={10}
onChange={(e) => {
setChunkLen(+e);
setReShowRePreview(true);
}}
>
<NumberInputField />
<NumberInputStepper>
<NumberIncrementStepper />
<NumberDecrementStepper />
</NumberInputStepper>
</NumberInput>
</MyTooltip>
</Box>
</Box>
{/* custom split char */}
<Box mt={4} alignItems={'center'}>
<Box>
{t('core.dataset.import.Custom split char')}
<MyTooltip label={t('core.dataset.import.Custom split char Tips')} forceShow>
<QuestionOutlineIcon />
</MyTooltip>
</Box>
<Box mt={1}>
<Input
defaultValue={''}
placeholder="\n;======;==SPLIT=="
onChange={(e) => {
setCustomSplitChar(e.target.value);
setReShowRePreview(true);
}}
/>
</Box>
</Box>
<Grid mt={4} gridTemplateColumns={'1fr 1fr'} gridGap={2}>
<Flex alignItems={'center'}>
<Box>{t('core.dataset.import.Total tokens')}：</Box>
<Box>{totalTokens}</Box>
</Flex>
{/* price */}
<Flex alignItems={'center'}>
<Box>
{t('core.dataset.import.Estimated Price')}
<MyTooltip
label={t('core.dataset.import.Embedding Estimated Price Tips', {
price: unitPrice
})}
forceShow
>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
</Flex>
</Grid>
<Flex mt={3}>
{showRePreview && (
<Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
{t('core.dataset.import.Re Preview')}
</Button>
)}
<Button
isDisabled={uploading}
onClick={() => {
onReSplitChunks();
openConfirm(onclickUpload)();
}}
>
{uploading ? (
<Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
) : (
t('common.Confirm Import')
)}
</Button>
</Flex>
</SelectorContainer>
{!isUnselectedFile && (
<Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
<PreviewFileOrChunk />
</Box>
)}
<ConfirmModal />
</Box>
);
};
export default ChunkImport;

72
projects/app/src/pages/dataset/detail/components/Import/CreateFileModal.tsx
View File

@@ -1,72 +0,0 @@
import React from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Input, Textarea, ModalBody, ModalFooter, Button } from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
import { useRequest } from '@/web/common/hooks/useRequest';
const CreateFileModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: { filename: string; content: string }) => Promise<void>;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
filename: '',
content: ''
}
});
const { mutate, isLoading } = useRequest({
mutationFn: () => handleSubmit(onSuccess)(),
onSuccess: () => {
onClose();
}
});
return (
<MyModal
title={t('file.Create File')}
iconSrc="/imgs/modal/txt.svg"
isOpen
w={'600px'}
top={'15vh'}
>
<ModalBody>
<Box mb={1} fontSize={'sm'}>
{t('common.file.File Name')}
</Box>
<Input
mb={5}
{...register('filename', {
required: t('common.file.Filename Can not Be Empty')
})}
/>
<Box mb={1} fontSize={'sm'}>
{t('common.file.File Content')}
</Box>
<Textarea
{...register('content', {
required: t('common.file.File content can not be empty')
})}
rows={12}
whiteSpace={'nowrap'}
resize={'both'}
/>
</ModalBody>
<ModalFooter>
<Button variant={'whiteBase'} mr={4} onClick={onClose}>
{t('common.Close')}
</Button>
<Button isLoading={isLoading} onClick={mutate}>
{t('common.Confirm Create')}
</Button>
</ModalFooter>
</MyModal>
);
};
export default CreateFileModal;

90
projects/app/src/pages/dataset/detail/components/Import/Csv.tsx
View File

@@ -1,90 +0,0 @@
import React from 'react';
import { Box, Flex, Button, Grid } from '@chakra-ui/react';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
import { useTranslation } from 'next-i18next';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
const fileExtension = '.csv';
const csvTemplate = `index,content
"必填内容","可选内容。CSV 中请注意内容不能包含双引号，双引号是列分割符号"
"结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。",""
"AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`;
const CsvImport = () => {
const { t } = useTranslation();
const {
successChunks,
totalChunks,
isUnselectedFile,
onclickUpload,
uploading,
totalTokens,
price
} = useImportStore();
const { datasetDetail } = useDatasetStore();
const vectorModel = datasetDetail.vectorModel;
const unitPrice = vectorModel?.inputPrice || 0.002;
const { openConfirm, ConfirmModal } = useConfirm({
content: t('core.dataset.import.Import Tip')
});
return (
<Box display={['block', 'flex']} h={['auto', '100%']}>
<SelectorContainer
fileExtension={fileExtension}
showUrlFetch={false}
fileTemplate={{
filename: 'csv templates.csv',
value: csvTemplate,
type: 'text/csv'
}}
tip={t('dataset.import csv tip')}
>
<Grid mt={4} gridTemplateColumns={'1fr 1fr'} gridGap={2}>
<Flex alignItems={'center'}>
<Box>{t('core.dataset.import.Total tokens')}：</Box>
<Box>{totalTokens}</Box>
</Flex>
{/* price */}
<Flex alignItems={'center'}>
<Box>
{t('core.dataset.import.Estimated Price')}
<MyTooltip
label={t('core.dataset.import.Embedding Estimated Price Tips', {
price: unitPrice
})}
forceShow
>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
</Flex>
</Grid>
<Flex mt={3}>
<Button isDisabled={uploading} onClick={openConfirm(onclickUpload)}>
{uploading ? (
<Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
) : (
t('common.Confirm Import')
)}
</Button>
</Flex>
</SelectorContainer>
{!isUnselectedFile && (
<Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
<PreviewFileOrChunk />
</Box>
)}
<ConfirmModal />
</Box>
);
};
export default CsvImport;

446
projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx
View File

@@ -1,446 +0,0 @@
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useLoading } from '@/web/common/hooks/useLoading';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useToast } from '@/web/common/hooks/useToast';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { simpleText } from '@fastgpt/global/common/string/tools';
import { fileDownload, readCsvContent } from '@/web/common/file/utils';
import { getUploadBase64ImgController, uploadFiles } from '@/web/common/file/controller';
import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react';
import React, { DragEvent, useCallback, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { customAlphabet } from 'nanoid';
import dynamic from 'next/dynamic';
import MyTooltip from '@/components/MyTooltip';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d';
import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
import { readFileRawContent } from '@fastgpt/web/common/file/read/index';
import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
const CreateFileModal = dynamic(() => import('./CreateFileModal'));
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
export type FileItemType = {
id: string; // fileId / raw Link
filename: string;
chunks: PushDatasetDataChunkProps[];
rawText: string; // raw text
icon: string;
tokens: number; // total tokens
type: DatasetCollectionTypeEnum.file | DatasetCollectionTypeEnum.link;
fileId?: string;
rawLink?: string;
metadata?: Record<string, any>;
};
export interface Props extends BoxProps {
fileExtension: string;
onPushFiles: (files: FileItemType[]) => void;
tipText?: string;
chunkLen?: number;
customSplitChar?: string;
overlapRatio?: number;
fileTemplate?: {
type: string;
filename: string;
value: string;
};
showUrlFetch?: boolean;
showCreateFile?: boolean;
tip?: string;
}
const FileSelect = ({
fileExtension,
onPushFiles,
tipText,
chunkLen = 500,
customSplitChar,
overlapRatio,
fileTemplate,
showUrlFetch = true,
showCreateFile = true,
tip,
...props
}: Props) => {
const { datasetDetail } = useDatasetStore();
const { Loading: FileSelectLoading } = useLoading();
const { t } = useTranslation();
const { toast } = useToast();
const { File: FileSelector, onOpen } = useSelectFile({
fileType: fileExtension,
multiple: true
});
const [isDragging, setIsDragging] = useState(false);
const [selectingText, setSelectingText] = useState<string>();
const {
isOpen: isOpenUrlFetch,
onOpen: onOpenUrlFetch,
onClose: onCloseUrlFetch
} = useDisclosure();
const {
isOpen: isOpenCreateFile,
onOpen: onOpenCreateFile,
onClose: onCloseCreateFile
} = useDisclosure();
// select file
const onSelectFile = useCallback(
async (files: File[]) => {
if (files.length >= 100) {
return toast({
status: 'warning',
title: t('common.file.Select file amount limit 100')
});
}
try {
for await (let file of files) {
const extension = file?.name?.split('.')?.pop()?.toLowerCase();
/* text file */
const icon = getFileIcon(file?.name);
// ts
if (!icon) continue;
// upload file
const filesId = await uploadFiles({
files: [file],
bucketName: 'dataset',
metadata: { datasetId: datasetDetail._id },
percentListen: (percent) => {
if (percent < 100) {
setSelectingText(
t('file.Uploading', { name: file.name.slice(0, 30), percent }) || ''
);
} else {
setSelectingText(t('file.Parse', { name: file.name.slice(0, 30) }) || '');
}
}
});
const fileId = filesId[0];
/* QA csv file */
if (extension === 'csv') {
const { header, data } = await readCsvContent(file);
if (header[0] !== 'index' || header[1] !== 'content') {
throw new Error(t('core.dataset.import.Csv format error'));
}
const filterData = data
.filter((item) => item[0])
.map((item) => ({
q: item[0] || '',
a: item[1] || ''
}));
const fileItem: FileItemType = {
id: nanoid(),
filename: file.name,
icon,
tokens: filterData.reduce((sum, item) => sum + countPromptTokens(item.q), 0),
rawText: `${header.join(',')}\n${data
.map((item) => `"${item[0]}","${item[1]}"`)
.join('\n')}`,
chunks: filterData,
type: DatasetCollectionTypeEnum.file,
fileId
};
onPushFiles([fileItem]);
continue;
}
// parse and upload files
let { rawText } = await readFileRawContent({
file,
uploadBase64Controller: (base64Img) =>
getUploadBase64ImgController({
base64Img,
type: MongoImageTypeEnum.docImage,
metadata: {
fileId
}
})
});
if (rawText) {
rawText = simpleText(rawText);
const { chunks, tokens } = splitText2Chunks({
text: rawText,
chunkLen,
overlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
const fileItem: FileItemType = {
id: nanoid(),
filename: file.name,
icon,
rawText,
tokens,
type: DatasetCollectionTypeEnum.file,
fileId,
chunks: chunks.map((chunk) => ({
q: chunk,
a: ''
}))
};
onPushFiles([fileItem]);
}
}
} catch (error: any) {
console.log(error);
toast({
title: getErrText(error, t('common.file.Read File Error')),
status: 'error'
});
}
setSelectingText(undefined);
},
[chunkLen, customSplitChar, datasetDetail._id, onPushFiles, overlapRatio, t, toast]
);
// link fetch
const onUrlFetch = useCallback(
(e: UrlFetchResponse) => {
const result: FileItemType[] = e.map<FileItemType>(({ url, content, selector }) => {
const { chunks, tokens } = splitText2Chunks({
text: content,
chunkLen,
overlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
return {
id: nanoid(),
filename: url,
icon: '/imgs/files/link.svg',
rawText: content,
tokens,
type: DatasetCollectionTypeEnum.link,
rawLink: url,
chunks: chunks.map((chunk) => ({
q: chunk,
a: ''
})),
metadata: {
webPageSelector: selector
}
};
});
onPushFiles(result);
},
[chunkLen, customSplitChar, onPushFiles, overlapRatio]
);
// manual create file and copy data
const onCreateFile = useCallback(
async ({ filename, content }: { filename: string; content: string }) => {
content = simpleText(content);
// create virtual txt file
const txtBlob = new Blob([content], { type: 'text/plain' });
const txtFile = new File([txtBlob], `${filename}.txt`, {
type: txtBlob.type,
lastModified: new Date().getTime()
});
const fileIds = await uploadFiles({
files: [txtFile],
bucketName: 'dataset',
metadata: { datasetId: datasetDetail._id }
});
const { chunks, tokens } = splitText2Chunks({
text: content,
chunkLen,
overlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
onPushFiles([
{
id: nanoid(),
filename,
icon: '/imgs/files/txt.svg',
rawText: content,
tokens,
type: DatasetCollectionTypeEnum.file,
fileId: fileIds[0],
chunks: chunks.map((chunk) => ({
q: chunk,
a: ''
}))
}
]);
},
[chunkLen, customSplitChar, datasetDetail._id, onPushFiles, overlapRatio]
);
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(true);
};
const handleDragLeave = (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(false);
};
const handleDrop = useCallback(
async (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(false);
const items = e.dataTransfer.items;
const fileList: File[] = [];
if (e.dataTransfer.items.length <= 1) {
const traverseFileTree = async (item: any) => {
return new Promise<void>((resolve, reject) => {
if (item.isFile) {
item.file((file: File) => {
fileList.push(file);
resolve();
});
} else if (item.isDirectory) {
const dirReader = item.createReader();
dirReader.readEntries(async (entries: any[]) => {
for (let i = 0; i < entries.length; i++) {
await traverseFileTree(entries[i]);
}
resolve();
});
}
});
};
for (let i = 0; i < items.length; i++) {
const item = items[i].webkitGetAsEntry();
if (item) {
await traverseFileTree(item);
}
}
} else {
const files = Array.from(e.dataTransfer.files);
let isErr = files.some((item) => item.type === '');
if (isErr) {
return toast({
title: t('file.upload error description'),
status: 'error'
});
}
for (let i = 0; i < files.length; i++) {
fileList.push(files[i]);
}
}
onSelectFile(fileList);
},
[onSelectFile, t, toast]
);
const SelectTextStyles: BoxProps = {
ml: 1,
as: 'span',
cursor: 'pointer',
color: 'primary.600',
_hover: {
textDecoration: 'underline'
}
};
return (
<Box
display={'inline-block'}
textAlign={'center'}
bg={'myWhite.400'}
p={5}
borderRadius={'md'}
border={'1px dashed'}
borderColor={'myGray.300'}
w={'100%'}
position={'relative'}
{...props}
onDragEnter={handleDragEnter}
onDragOver={handleDragEnter}
onDragLeave={handleDragLeave}
onDrop={handleDrop}
>
<Flex justifyContent={'center'} alignItems={'center'}>
<MyIcon mr={1} name={'file/uploadFile'} w={'16px'} />
{isDragging ? (
t('file.Release the mouse to upload the file')
) : (
<Box>
{t('file.Drag and drop')},
<MyTooltip label={t('file.max 10')}>
<Box {...SelectTextStyles} onClick={onOpen}>
{t('file.select a document')}
</Box>
</MyTooltip>
{showUrlFetch && (
<>
,
<Box {...SelectTextStyles} onClick={onOpenUrlFetch}>
{t('file.Fetch Url')}
</Box>
</>
)}
{showCreateFile && (
<>
,
<Box {...SelectTextStyles} onClick={onOpenCreateFile}>
{t('file.Create file')}
</Box>
</>
)}
</Box>
)}
</Flex>
<Box mt={1}>{t('file.support', { fileExtension: fileExtension })}</Box>
{tipText && (
<Box mt={1} fontSize={'sm'} color={'myGray.600'}>
{t(tipText)}
</Box>
)}
{!!fileTemplate && (
<Box
mt={1}
cursor={'pointer'}
textDecoration={'underline'}
color={'primary.500'}
fontSize={'12px'}
onClick={() =>
fileDownload({
text: fileTemplate.value,
type: fileTemplate.type,
filename: fileTemplate.filename
})
}
>
{t('file.Click to download file template', { name: fileTemplate.filename })}
</Box>
)}
{!!tip && <Box color={'myGray.500'}>{tip}</Box>}
{selectingText !== undefined && (
<FileSelectLoading loading text={selectingText} fixed={false} />
)}
<FileSelector onSelect={onSelectFile} />
{isOpenUrlFetch && <UrlFetchModal onClose={onCloseUrlFetch} onSuccess={onUrlFetch} />}
{isOpenCreateFile && <CreateFileModal onClose={onCloseCreateFile} onSuccess={onCreateFile} />}
</Box>
);
};
export default FileSelect;

138
projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx
View File

@@ -1,138 +0,0 @@
import React, { useMemo, useState } from 'react';
import { Box, type BoxProps, Flex, useTheme, ModalCloseButton } from '@chakra-ui/react';
import MyRadio from '@/components/common/MyRadio/index';
import dynamic from 'next/dynamic';
import ChunkImport from './Chunk';
import { useTranslation } from 'next-i18next';
const QAImport = dynamic(() => import('./QA'), {});
const CsvImport = dynamic(() => import('./Csv'), {});
import MyModal from '@/components/MyModal';
import Provider from './Provider';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
export enum ImportTypeEnum {
chunk = 'chunk',
qa = 'qa',
csv = 'csv'
}
const ImportData = ({
datasetId,
parentId,
onClose,
uploadSuccess
}: {
datasetId: string;
parentId: string;
onClose: () => void;
uploadSuccess: () => void;
}) => {
const { t } = useTranslation();
const theme = useTheme();
const { datasetDetail } = useDatasetStore();
const [importType, setImportType] = useState<`${ImportTypeEnum}`>(ImportTypeEnum.chunk);
const vectorModel = datasetDetail.vectorModel;
const agentModel = datasetDetail.agentModel;
const typeMap = useMemo(() => {
const map = {
[ImportTypeEnum.chunk]: {
defaultChunkLen: vectorModel?.defaultToken || 500,
chunkOverlapRatio: 0.2,
inputPrice: vectorModel?.inputPrice || 0,
outputPrice: 0,
collectionTrainingType: TrainingModeEnum.chunk
},
[ImportTypeEnum.qa]: {
defaultChunkLen: agentModel?.maxContext * 0.55 || 8000,
chunkOverlapRatio: 0,
inputPrice: agentModel?.inputPrice || 0,
outputPrice: agentModel?.outputPrice || 0,
collectionTrainingType: TrainingModeEnum.qa
},
[ImportTypeEnum.csv]: {
defaultChunkLen: 0,
chunkOverlapRatio: 0,
inputPrice: vectorModel?.inputPrice || 0,
outputPrice: 0,
collectionTrainingType: TrainingModeEnum.chunk
}
};
return map[importType];
}, [
agentModel?.inputPrice,
agentModel?.maxContext,
agentModel?.outputPrice,
importType,
vectorModel?.defaultToken,
vectorModel?.inputPrice
]);
const TitleStyle: BoxProps = {
fontWeight: 'bold',
fontSize: ['md', 'xl']
};
return (
<MyModal
iconSrc="/imgs/modal/import.svg"
title={<Box {...TitleStyle}>{t('dataset.data.File import')}</Box>}
isOpen
isCentered
maxW={['90vw', 'min(1440px,85vw)']}
w={['90vw', 'min(1440px,85vw)']}
h={'90vh'}
>
<ModalCloseButton onClick={onClose} />
<Flex mt={2} flexDirection={'column'} flex={'1 0 0'}>
<Box pb={[5, 7]} px={[4, 8]} borderBottom={theme.borders.base}>
<MyRadio
gridTemplateColumns={['repeat(1,1fr)', 'repeat(3,1fr)']}
list={[
{
icon: 'file/indexImport',
title: t('core.dataset.import.Chunk Split'),
desc: t('core.dataset.import.Chunk Split Tip'),
value: ImportTypeEnum.chunk
},
{
icon: 'file/qaImport',
title: t('core.dataset.import.QA Import'),
desc: t('core.dataset.import.QA Import Tip'),
value: ImportTypeEnum.qa
},
{
icon: 'file/csv',
title: t('core.dataset.import.CSV Import'),
desc: t('core.dataset.import.CSV Import Tip'),
value: ImportTypeEnum.csv
}
]}
value={importType}
onChange={(e) => setImportType(e as `${ImportTypeEnum}`)}
/>
</Box>
<Provider
{...typeMap}
vectorModel={vectorModel.model}
agentModel={agentModel.model}
datasetId={datasetDetail._id}
importType={importType}
parentId={parentId}
onUploadSuccess={uploadSuccess}
>
<Box flex={'1 0 0'} h={0}>
{importType === ImportTypeEnum.chunk && <ChunkImport />}
{importType === ImportTypeEnum.qa && <QAImport />}
{importType === ImportTypeEnum.csv && <CsvImport />}
</Box>
</Provider>
</Flex>
</MyModal>
);
};
export default ImportData;

629
projects/app/src/pages/dataset/detail/components/Import/Provider.tsx
View File

@@ -1,502 +1,207 @@
import React, {
type SetStateAction,
type Dispatch,
useContext,
useCallback,
createContext,
useState,
useMemo,
useEffect
} from 'react';
import FileSelect, { FileItemType, Props as FileSelectProps } from './FileSelect';
import { useRequest } from '@/web/common/hooks/useRequest';
import { postDatasetCollection } from '@/web/core/dataset/api';
import React, { useContext, useCallback, createContext, useState, useMemo, useEffect } from 'react';
import { formatModelPrice2Read } from '@fastgpt/global/support/wallet/bill/tools';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { hashStr } from '@fastgpt/global/common/string/tools';
import { useToast } from '@/web/common/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
import { Box, Flex, Image, useTheme } from '@chakra-ui/react';
import { CloseIcon } from '@chakra-ui/icons';
import DeleteIcon, { hoverDeleteStyles } from '@fastgpt/web/components/common/Icon/delete';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { chunksUpload } from '@/web/core/dataset/utils';
import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { useTranslation } from 'next-i18next';
import { ImportTypeEnum } from './ImportModal';
import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
import { Prompt_AgentQA } from '@/global/core/prompt/agent';
import { UseFormReturn, useForm } from 'react-hook-form';
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
import { ImportSourceItemType } from '@/web/core/dataset/type';
const filenameStyles = {
className: 'textEllipsis',
maxW: '400px'
type ChunkSizeFieldType = 'embeddingChunkSize';
export type FormType = {
mode: `${TrainingModeEnum}`;
way: `${ImportProcessWayEnum}`;
embeddingChunkSize: number;
customSplitChar: string;
qaPrompt: string;
webSelector: string;
};
type useImportStoreType = {
files: FileItemType[];
setFiles: Dispatch<SetStateAction<FileItemType[]>>;
previewFile: FileItemType | undefined;
setPreviewFile: Dispatch<SetStateAction<FileItemType | undefined>>;
successChunks: number;
setSuccessChunks: Dispatch<SetStateAction<number>>;
isUnselectedFile: boolean;
totalChunks: number;
totalTokens: number;
onclickUpload: (e?: { prompt?: string }) => void;
onReSplitChunks: () => void;
price: number;
uploading: boolean;
chunkLen: number;
chunkOverlapRatio: number;
setChunkLen: Dispatch<number>;
customSplitChar?: string;
setCustomSplitChar: Dispatch<string>;
parentId?: string;
processParamsForm: UseFormReturn<FormType, any>;
chunkSizeField?: ChunkSizeFieldType;
maxChunkSize: number;
minChunkSize: number;
showChunkInput: boolean;
showPromptInput: boolean;
sources: ImportSourceItemType[];
setSources: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
showRePreview: boolean;
setReShowRePreview: Dispatch<SetStateAction<boolean>>;
totalChunkChars: number;
totalChunks: number;
chunkSize: number;
predictPrice: number;
priceTip: string;
uploadRate: number;
splitSources2Chunks: () => void;
};
const StateContext = createContext<useImportStoreType>({
onclickUpload: function (e?: { prompt?: string }): void {
throw new Error('Function not implemented.');
},
uploading: false,
files: [],
previewFile: undefined,
successChunks: 0,
isUnselectedFile: false,
totalChunks: 0,
totalTokens: 0,
onReSplitChunks: function (): void {
throw new Error('Function not implemented.');
},
price: 0,
chunkLen: 0,
chunkOverlapRatio: 0,
customSplitChar: undefined,
setCustomSplitChar: function (value: string): void {
throw new Error('Function not implemented.');
},
setChunkLen: function (value: number): void {
throw new Error('Function not implemented.');
},
setFiles: function (value: React.SetStateAction<FileItemType[]>): void {
throw new Error('Function not implemented.');
},
setPreviewFile: function (value: React.SetStateAction<FileItemType | undefined>): void {
throw new Error('Function not implemented.');
},
setSuccessChunks: function (value: React.SetStateAction<number>): void {
processParamsForm: {} as any,
sources: [],
setSources: function (value: React.SetStateAction<ImportSourceItemType[]>): void {
throw new Error('Function not implemented.');
},
maxChunkSize: 0,
minChunkSize: 0,
showChunkInput: false,
showPromptInput: false,
chunkSizeField: 'embeddingChunkSize',
showRePreview: false,
setReShowRePreview: function (value: React.SetStateAction<boolean>): void {
throw new Error('Function not implemented.');
}
totalChunkChars: 0,
totalChunks: 0,
chunkSize: 0,
predictPrice: 0,
priceTip: '',
uploadRate: 50,
splitSources2Chunks: () => {}
});
export const useImportStore = () => useContext(StateContext);
const Provider = ({
datasetId,
dataset,
parentId,
inputPrice,
outputPrice,
collectionTrainingType,
vectorModel,
agentModel,
defaultChunkLen = 500,
chunkOverlapRatio = 0.2,
importType,
onUploadSuccess,
children
}: {
datasetId: string;
parentId: string;
inputPrice: number;
outputPrice: number;
collectionTrainingType: `${TrainingModeEnum}`;
vectorModel: string;
agentModel: string;
defaultChunkLen: number;
chunkOverlapRatio: number;
importType: `${ImportTypeEnum}`;
onUploadSuccess: () => void;
dataset: DatasetItemType;
parentId?: string;
children: React.ReactNode;
}) => {
const vectorModel = dataset.vectorModel;
const agentModel = dataset.agentModel;
const processParamsForm = useForm<FormType>({
defaultValues: {
mode: TrainingModeEnum.chunk,
way: ImportProcessWayEnum.auto,
embeddingChunkSize: vectorModel?.defaultToken || 512,
customSplitChar: '',
qaPrompt: Prompt_AgentQA.description,
webSelector: ''
}
});
const { t } = useTranslation();
const { toast } = useToast();
const [files, setFiles] = useState<FileItemType[]>([]);
const [successChunks, setSuccessChunks] = useState(0);
const [chunkLen, setChunkLen] = useState(defaultChunkLen);
const [customSplitChar, setCustomSplitChar] = useState<string>();
const [previewFile, setPreviewFile] = useState<FileItemType>();
const [showRePreview, setReShowRePreview] = useState(false);
const [sources, setSources] = useState<ImportSourceItemType[]>([]);
const [showRePreview, setShowRePreview] = useState(false);
const isUnselectedFile = useMemo(() => files.length === 0, [files]);
// watch form
const mode = processParamsForm.watch('mode');
const way = processParamsForm.watch('way');
const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize');
const customSplitChar = processParamsForm.watch('customSplitChar');
const totalChunks = useMemo(
() => files.reduce((sum, file) => sum + file.chunks.length, 0),
[files]
const modeStaticParams = {
[TrainingModeEnum.chunk]: {
chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType,
chunkOverlapRatio: 0.2,
maxChunkSize: vectorModel?.maxToken || 512,
minChunkSize: 100,
autoChunkSize: vectorModel?.defaultToken || 512,
chunkSize: embeddingChunkSize,
showChunkInput: true,
showPromptInput: false,
inputPrice: vectorModel.inputPrice,
outputPrice: 0,
priceTip: t('core.dataset.import.Embedding Estimated Price Tips', {
price: vectorModel.inputPrice
}),
uploadRate: 150
},
[TrainingModeEnum.qa]: {
chunkOverlapRatio: 0,
maxChunkSize: 8000,
minChunkSize: 3000,
autoChunkSize: agentModel.maxContext * 0.55 || 6000,
chunkSize: agentModel.maxContext * 0.55 || 6000,
showChunkInput: false,
showPromptInput: true,
inputPrice: agentModel.inputPrice,
outputPrice: agentModel.outputPrice,
priceTip: t('core.dataset.import.QA Estimated Price Tips', {
price: agentModel?.inputPrice
}),
uploadRate: 30
}
};
const selectModelStaticParam = useMemo(() => modeStaticParams[mode], [mode]);
const wayStaticPrams = {
[ImportProcessWayEnum.auto]: {
chunkSize: selectModelStaticParam.autoChunkSize,
customSplitChar: ''
},
[ImportProcessWayEnum.custom]: {
chunkSize: modeStaticParams[mode].chunkSize,
customSplitChar
}
};
const chunkSize = wayStaticPrams[way].chunkSize;
useEffect(() => {
setShowRePreview(true);
}, [mode, way, chunkSize, customSplitChar]);
const totalChunkChars = useMemo(
() => sources.reduce((sum, file) => sum + file.chunkChars, 0),
[sources]
);
const totalTokens = useMemo(() => files.reduce((sum, file) => sum + file.tokens, 0), [files]);
const price = useMemo(() => {
if (collectionTrainingType === TrainingModeEnum.qa) {
const inputTotal = totalTokens * inputPrice;
const outputTotal = totalTokens * 0.5 * outputPrice;
const predictPrice = useMemo(() => {
if (mode === TrainingModeEnum.qa) {
const inputTotal = totalChunkChars * selectModelStaticParam.inputPrice;
const outputTotal = totalChunkChars * 0.5 * selectModelStaticParam.inputPrice;
return formatModelPrice2Read(inputTotal + outputTotal);
}
return formatModelPrice2Read(totalTokens * inputPrice);
}, [collectionTrainingType, inputPrice, outputPrice, totalTokens]);
return formatModelPrice2Read(totalChunkChars * selectModelStaticParam.inputPrice);
}, [mode, selectModelStaticParam.inputPrice, totalChunkChars]);
const totalChunks = useMemo(
() => sources.reduce((sum, file) => sum + file.chunks.length, 0),
[sources]
);
/*
start upload data
1. create training bill
2. create collection
3. upload chunks
*/
const { mutate: onclickUpload, isLoading: uploading } = useRequest({
mutationFn: async (props?: { prompt?: string }) => {
const { prompt } = props || {};
let totalInsertion = 0;
for await (const file of files) {
// create training bill
const billId = await postCreateTrainingBill({
name: file.filename,
vectorModel,
agentModel
const splitSources2Chunks = useCallback(() => {
setSources((state) =>
state.map((file) => {
const { chunks, chars } = splitText2Chunks({
text: file.rawText,
chunkLen: chunkSize,
overlapRatio: selectModelStaticParam.chunkOverlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
// create a file collection and training bill
const collectionId = await postDatasetCollection({
datasetId,
parentId,
name: file.filename,
type: file.type,
trainingType: collectionTrainingType,
chunkSize: chunkLen,
chunkSplitter: customSplitChar,
qaPrompt: collectionTrainingType === TrainingModeEnum.qa ? prompt : '',
fileId: file.fileId,
rawLink: file.rawLink,
rawTextLength: file.rawText.length,
hashRawText: hashStr(file.rawText),
metadata: file.metadata
});
// upload chunks
const chunks = file.chunks;
const { insertLen } = await chunksUpload({
collectionId,
billId,
trainingMode: collectionTrainingType,
chunks,
onUploading: (insertLen) => {
setSuccessChunks((state) => state + insertLen);
},
prompt
});
totalInsertion += insertLen;
}
return totalInsertion;
},
onSuccess(num) {
toast({
title: t('core.dataset.import.Import Success Tip', { num }),
status: 'success'
});
onUploadSuccess();
},
errorToast: t('core.dataset.import.Import Failed')
});
const onReSplitChunks = useCallback(async () => {
try {
setPreviewFile(undefined);
setFiles((state) =>
state.map((file) => {
const { chunks, tokens } = splitText2Chunks({
text: file.rawText,
chunkLen,
overlapRatio: chunkOverlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
return {
...file,
tokens,
chunks: chunks.map((chunk) => ({
q: chunk,
a: ''
}))
};
})
);
setReShowRePreview(false);
} catch (error) {
toast({
status: 'warning',
title: getErrText(error, t('core.dataset.import.Set Chunk Error'))
});
}
}, [chunkLen, chunkOverlapRatio, customSplitChar, t, toast]);
const reset = useCallback(() => {
setFiles([]);
setSuccessChunks(0);
setChunkLen(defaultChunkLen);
setPreviewFile(undefined);
setReShowRePreview(false);
}, [defaultChunkLen]);
useEffect(() => {
reset();
}, [importType, reset]);
return {
...file,
chunkChars: chars,
chunks: chunks.map((chunk) => ({
q: chunk,
a: ''
}))
};
})
);
setShowRePreview(false);
}, [chunkSize, customSplitChar, selectModelStaticParam.chunkOverlapRatio]);
const value = {
files,
setFiles,
previewFile,
setPreviewFile,
successChunks,
setSuccessChunks,
isUnselectedFile,
totalChunks,
totalTokens,
price,
onReSplitChunks,
onclickUpload,
uploading,
chunkLen,
customSplitChar,
setCustomSplitChar,
chunkOverlapRatio,
setChunkLen,
parentId,
processParamsForm,
...selectModelStaticParam,
sources,
setSources,
showRePreview,
setReShowRePreview
totalChunkChars,
totalChunks,
chunkSize,
predictPrice,
splitSources2Chunks
};
return <StateContext.Provider value={value}>{children}</StateContext.Provider>;
};
export default React.memo(Provider);
export const PreviewFileOrChunk = () => {
const theme = useTheme();
const { t } = useTranslation();
const { setFiles, previewFile, setPreviewFile, setReShowRePreview, totalChunks, files } =
useImportStore();
return (
<Box h={'100%'} w={'100%'}>
{!!previewFile ? (
<Box
position={'relative'}
display={['block', 'flex']}
h={'100%'}
flexDirection={'column'}
pt={[3, 6]}
bg={'myWhite.400'}
>
<Box px={[4, 8]} fontSize={['lg', 'xl']} fontWeight={'bold'} {...filenameStyles}>
{previewFile.filename}
</Box>
<CloseIcon
position={'absolute'}
right={[4, 8]}
top={4}
cursor={'pointer'}
onClick={() => setPreviewFile(undefined)}
/>
<Box
flex={'1 0 0'}
h={['auto', 0]}
overflow={'overlay'}
px={[4, 8]}
my={4}
// contentEditable
// dangerouslySetInnerHTML={{ __html: previewFile.rawText }}
fontSize={'sm'}
whiteSpace={'pre-wrap'}
wordBreak={'break-all'}
// onBlur={(e) => {
// // @ts-ignore
// const val = e.target.innerText;
// setReShowRePreview(true);
// setFiles((state) =>
// state.map((file) =>
// file.id === previewFile.id
// ? {
// ...file,
// text: val
// }
// : file
// )
// );
// }}
>
{previewFile.rawText}
</Box>
</Box>
) : (
<Box pt={[3, 6]}>
<Flex px={[4, 8]} alignItems={'center'}>
<Box fontSize={['lg', 'xl']} fontWeight={'bold'}>
{t('core.dataset.import.Total Chunk Preview', { totalChunks })}
</Box>
{totalChunks > 50 && (
<Box ml={2} fontSize={'sm'} color={'myhGray.500'}>
{t('core.dataset.import.Only Show First 50 Chunk')}
</Box>
)}
</Flex>
<Box px={[4, 8]} overflow={'overlay'}>
{files.map((file) =>
file.chunks.slice(0, 50).map((chunk, i) => (
<Box
key={i}
py={4}
bg={'myWhite.500'}
my={2}
borderRadius={'md'}
fontSize={'sm'}
_hover={{ ...hoverDeleteStyles }}
>
<Flex mb={1} px={4} userSelect={'none'}>
<Box
flexShrink={0}
px={3}
py={'1px'}
border={theme.borders.base}
borderRadius={'md'}
>
# {i + 1}
</Box>
<Box ml={2} fontSize={'sm'} color={'myhGray.500'} {...filenameStyles}>
{file.filename}
</Box>
<Box flex={1} />
<DeleteIcon
onClick={() => {
setFiles((state) =>
state.map((stateFile) =>
stateFile.id === file.id
? {
...file,
chunks: [...file.chunks.slice(0, i), ...file.chunks.slice(i + 1)]
}
: stateFile
)
);
}}
/>
</Flex>
<Box px={4} fontSize={'sm'} whiteSpace={'pre-wrap'} wordBreak={'break-all'}>
{chunk.a ? `q:${chunk.q}\na:${chunk.a}` : chunk.q}
</Box>
</Box>
))
)}
</Box>
</Box>
)}
</Box>
);
};
export const SelectorContainer = ({
fileExtension,
showUrlFetch,
showCreateFile,
fileTemplate,
tip,
children
}: {
fileExtension: string;
showUrlFetch?: boolean;
showCreateFile?: boolean;
fileTemplate?: FileSelectProps['fileTemplate'];
tip?: string;
children: React.ReactNode;
}) => {
const { files, setPreviewFile, isUnselectedFile, setFiles, chunkLen, chunkOverlapRatio } =
useImportStore();
return (
<Box
h={'100%'}
overflowY={'auto'}
flex={['auto', '1 0 400px']}
{...(isUnselectedFile
? {}
: {
maxW: ['auto', '450px']
})}
p={[4, 8]}
>
<FileSelect
fileExtension={fileExtension}
onPushFiles={(files) => {
setFiles((state) => files.concat(state));
}}
chunkLen={chunkLen}
overlapRatio={chunkOverlapRatio}
showUrlFetch={showUrlFetch}
showCreateFile={showCreateFile}
fileTemplate={fileTemplate}
tip={tip}
py={isUnselectedFile ? '100px' : 5}
/>
{!isUnselectedFile && (
<Box py={4} px={2} maxH={'400px'} overflowY={'auto'}>
{files.map((item) => (
<Flex
key={item.id}
w={'100%'}
_notLast={{ mb: 5 }}
px={5}
py={2}
boxShadow={'1px 1px 5px rgba(0,0,0,0.15)'}
borderRadius={'md'}
cursor={'pointer'}
position={'relative'}
alignItems={'center'}
_hover={{
bg: 'primary.50',
'& .delete': {
display: 'block'
}
}}
onClick={() => setPreviewFile(item)}
>
<Image src={item.icon} w={'16px'} alt={''} />
<Box ml={2} flex={'1 0 0'} pr={3} {...filenameStyles}>
{item.filename}
</Box>
<MyIcon
position={'absolute'}
right={3}
className="delete"
name={'delete'}
w={'16px'}
_hover={{ color: 'red.600' }}
display={['block', 'none']}
onClick={(e) => {
e.stopPropagation();
setPreviewFile(undefined);
setFiles((state) => state.filter((file) => file.id !== item.id));
}}
/>
</Flex>
))}
</Box>
)}
{!isUnselectedFile && <>{children}</>}
</Box>
);
};

112
projects/app/src/pages/dataset/detail/components/Import/QA.tsx
View File

@@ -1,112 +0,0 @@
import React, { useState } from 'react';
import { Box, Flex, Button, Textarea, Grid } from '@chakra-ui/react';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { Prompt_AgentQA } from '@/global/core/prompt/agent';
import { useImportStore, SelectorContainer, PreviewFileOrChunk } from './Provider';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { useTranslation } from 'next-i18next';
const fileExtension = '.txt, .docx, .pdf, .md, .html';
const QAImport = () => {
const { t } = useTranslation();
const { datasetDetail } = useDatasetStore();
const agentModel = datasetDetail.agentModel;
const {
successChunks,
totalChunks,
totalTokens,
isUnselectedFile,
price,
onclickUpload,
onReSplitChunks,
uploading,
showRePreview
} = useImportStore();
const { openConfirm, ConfirmModal } = useConfirm({
content: t('core.dataset.import.Import Tip')
});
const [prompt, setPrompt] = useState(Prompt_AgentQA.description);
return (
<Box display={['block', 'flex']} h={['auto', '100%']}>
<SelectorContainer fileExtension={fileExtension}>
{/* prompt */}
<Box p={3} bg={'myWhite.600'} borderRadius={'md'}>
<Box mb={1} fontWeight={'bold'}>
{t('core.dataset.collection.QA Prompt')}
</Box>
<Box whiteSpace={'pre-wrap'} fontSize={'sm'}>
<Textarea
defaultValue={prompt}
rows={8}
fontSize={'sm'}
onChange={(e) => {
setPrompt(e.target.value);
}}
/>
<Box>{Prompt_AgentQA.fixedText}</Box>
</Box>
</Box>
{/* price */}
<Grid mt={4} gridTemplateColumns={'1fr 1fr'} gridGap={2}>
<Flex alignItems={'center'}>
<Box>{t('core.dataset.import.Total tokens')}：</Box>
<Box>{totalTokens}</Box>
</Flex>
{/* price */}
<Flex alignItems={'center'}>
<Box>
{t('core.dataset.import.Estimated Price')}
<MyTooltip
label={t('core.dataset.import.QA Estimated Price Tips', {
inputPrice: agentModel?.inputPrice,
outputPrice: agentModel?.outputPrice
})}
forceShow
>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Box ml={4}>{t('common.price.Amount', { amount: price, unit: '元' })}</Box>
</Flex>
</Grid>
<Flex mt={3}>
{showRePreview && (
<Button variant={'whitePrimary'} mr={4} onClick={onReSplitChunks}>
{t('core.dataset.import.Re Preview')}
</Button>
)}
<Button
isDisabled={uploading}
onClick={() => {
onReSplitChunks();
openConfirm(() => onclickUpload({ prompt }))();
}}
>
{uploading ? (
<Box>{Math.round((successChunks / totalChunks) * 100)}%</Box>
) : (
t('common.Confirm Import')
)}
</Button>
</Flex>
</SelectorContainer>
{!isUnselectedFile && (
<Box flex={['auto', '1 0 0']} h={'100%'} overflowY={'auto'}>
<PreviewFileOrChunk />
</Box>
)}
<ConfirmModal />
</Box>
);
};
export default QAImport;

94
projects/app/src/pages/dataset/detail/components/Import/UrlFetchModal.tsx
View File

@@ -1,94 +0,0 @@
import React from 'react';
import { useTranslation } from 'next-i18next';
import MyModal from '@/components/MyModal';
import { Box, Button, Input, Link, ModalBody, ModalFooter, Textarea } from '@chakra-ui/react';
import { useRequest } from '@/web/common/hooks/useRequest';
import { postFetchUrls } from '@/web/common/tools/api';
import { useForm } from 'react-hook-form';
import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
import { getDocPath } from '@/web/common/system/doc';
import { feConfigs } from '@/web/common/system/staticData';
const UrlFetchModal = ({
onClose,
onSuccess
}: {
onClose: () => void;
onSuccess: (e: UrlFetchResponse) => void;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
urls: '',
selector: ''
}
});
const { mutate, isLoading } = useRequest({
mutationFn: async ({ urls, selector }: { urls: string; selector: string }) => {
const urlList = urls.split('\n').filter((e) => e);
const res = await postFetchUrls({
urlList,
selector
});
onSuccess(res);
onClose();
},
errorToast: t('core.dataset.import.Fetch Error')
});
return (
<MyModal
iconSrc="/imgs/modal/network.svg"
title={
<Box>
<Box>{t('file.Fetch Url')}</Box>
<Box fontWeight={'normal'} fontSize={'sm'} color={'myGray.500'}>
{t('core.dataset.import.Fetch url tip')}
</Box>
</Box>
}
top={'15vh'}
isOpen
onClose={onClose}
w={'600px'}
>
<ModalBody>
<Box>
<Box fontWeight={'bold'}>{t('core.dataset.import.Fetch Url')}</Box>
<Textarea
{...register('urls', {
required: true
})}
rows={11}
whiteSpace={'nowrap'}
resize={'both'}
placeholder={t('core.dataset.import.Fetch url placeholder')}
/>
</Box>
<Box mt={4}>
<Box fontWeight={'bold'}>
{t('core.dataset.website.Selector')}({t('common.choosable')})
</Box>
{feConfigs?.docUrl && (
<Link href={getDocPath('/docs/course/websync/#选择器如何使用')} target="_blank">
{t('core.dataset.website.Selector Course')}
</Link>
)}
<Input {...register('selector')} placeholder="body .content #document" />
</Box>
</ModalBody>
<ModalFooter>
<Button variant={'whiteBase'} mr={4} onClick={onClose}>
{t('common.Close')}
</Button>
<Button isLoading={isLoading} onClick={handleSubmit((data) => mutate(data))}>
{t('common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};
export default UrlFetchModal;

356
projects/app/src/pages/dataset/detail/components/Import/commonProgress/DataProcess.tsx Normal file
View File

@@ -0,0 +1,356 @@
import React, { useEffect, useRef, useState } from 'react';
import {
Box,
Flex,
NumberInput,
NumberInputField,
NumberInputStepper,
NumberIncrementStepper,
NumberDecrementStepper,
Input,
Button,
ModalBody,
ModalFooter,
Textarea,
useDisclosure
} from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import { TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
import MyTooltip from '@/components/MyTooltip';
import { useImportStore } from '../Provider';
import { feConfigs } from '@/web/common/system/staticData';
import Tag from '@/components/Tag';
import MyModal from '@/components/MyModal';
import { Prompt_AgentQA } from '@/global/core/prompt/agent';
import Preview from '../components/Preview';
function DataProcess({
showPreviewChunks = true,
goToNext
}: {
showPreviewChunks: boolean;
goToNext: () => void;
}) {
const { t } = useTranslation();
const {
processParamsForm,
sources,
chunkSizeField,
minChunkSize,
showChunkInput,
showPromptInput,
maxChunkSize,
totalChunkChars,
totalChunks,
predictPrice,
showRePreview,
splitSources2Chunks,
priceTip
} = useImportStore();
const { getValues, setValue, register } = processParamsForm;
const [refresh, setRefresh] = useState(false);
const {
isOpen: isOpenCustomPrompt,
onOpen: onOpenCustomPrompt,
onClose: onCloseCustomPrompt
} = useDisclosure();
useEffect(() => {
if (showPreviewChunks) {
splitSources2Chunks();
}
}, []);
return (
<Box h={'100%'} display={['block', 'flex']} gap={5}>
<Box flex={'1 0 0'} maxW={'600px'}>
<Flex fontWeight={'bold'} alignItems={'center'}>
<MyIcon name={'common/settingLight'} w={'20px'} />
<Box fontSize={'lg'}>{t('core.dataset.import.Data process params')}</Box>
</Flex>
<Flex mt={4} alignItems={'center'}>
<Box color={'myGray.600'} flex={'0 0 100px'}>
{t('core.dataset.import.Training mode')}
</Box>
<LeftRadio
list={Object.entries(TrainingTypeMap).map(([key, value]) => ({
title: t(value.label),
value: key,
tooltip: t(value.tooltip)
}))}
px={3}
py={2}
value={getValues('mode')}
onChange={(e) => {
setValue('mode', e);
setRefresh(!refresh);
}}
gridTemplateColumns={'1fr 1fr'}
defaultBg="white"
activeBg="white"
/>
</Flex>
<Flex mt={5}>
<Box color={'myGray.600'} flex={'0 0 100px'}>
{t('core.dataset.import.Process way')}
</Box>
<LeftRadio
list={[
{
title: t('core.dataset.import.Auto process'),
desc: t('core.dataset.import.Auto process desc'),
value: ImportProcessWayEnum.auto
},
{
title: t('core.dataset.import.Custom process'),
desc: t('core.dataset.import.Custom process desc'),
value: ImportProcessWayEnum.custom,
children: getValues('way') === ImportProcessWayEnum.custom && (
<Box mt={5}>
{showChunkInput && chunkSizeField && (
<Box>
<Flex alignItems={'center'}>
<Box>{t('core.dataset.import.Ideal chunk length')}</Box>
<MyTooltip
label={t('core.dataset.import.Ideal chunk length Tips')}
forceShow
>
<MyIcon
name={'common/questionLight'}
ml={1}
w={'14px'}
color={'myGray.500'}
/>
</MyTooltip>
</Flex>
<Box
mt={1}
css={{
'& > span': {
display: 'block'
}
}}
>
<MyTooltip
label={t('core.dataset.import.Chunk Range', {
min: minChunkSize,
max: maxChunkSize
})}
>
<NumberInput
size={'sm'}
step={100}
min={minChunkSize}
max={maxChunkSize}
onChange={(e) => {
setValue(chunkSizeField, +e);
}}
>
<NumberInputField
min={minChunkSize}
max={maxChunkSize}
{...register(chunkSizeField, {
min: minChunkSize,
max: maxChunkSize,
valueAsNumber: true
})}
/>
<NumberInputStepper>
<NumberIncrementStepper />
<NumberDecrementStepper />
</NumberInputStepper>
</NumberInput>
</MyTooltip>
</Box>
</Box>
)}
<Box mt={3}>
<Box>
{t('core.dataset.import.Custom split char')}
<MyTooltip
label={t('core.dataset.import.Custom split char Tips')}
forceShow
>
<MyIcon
name={'common/questionLight'}
ml={1}
w={'14px'}
color={'myGray.500'}
/>
</MyTooltip>
</Box>
<Box mt={1}>
<Input
size={'sm'}
bg={'myGray.50'}
defaultValue={''}
placeholder="\n;======;==SPLIT=="
{...register('customSplitChar')}
/>
</Box>
</Box>
{showPromptInput && (
<Box mt={3}>
<Box>{t('core.dataset.collection.QA Prompt')}</Box>
<Box
position={'relative'}
py={2}
px={3}
bg={'myGray.50'}
fontSize={'xs'}
whiteSpace={'pre-wrap'}
border={'1px'}
borderColor={'borderColor.base'}
borderRadius={'md'}
maxH={'140px'}
overflow={'auto'}
_hover={{
'& .mask': {
display: 'block'
}
}}
>
{getValues('qaPrompt')}
<Box
display={'none'}
className="mask"
position={'absolute'}
top={0}
right={0}
bottom={0}
left={0}
background={
'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
}
>
<Button
size="xs"
variant={'whiteBase'}
leftIcon={<MyIcon name={'edit'} w={'13px'} />}
color={'black'}
position={'absolute'}
right={2}
bottom={2}
onClick={onOpenCustomPrompt}
>
{t('core.dataset.import.Custom prompt')}
</Button>
</Box>
</Box>
</Box>
)}
</Box>
)
}
]}
px={3}
py={3}
defaultBg="white"
activeBg="white"
value={getValues('way')}
w={'100%'}
onChange={(e) => {
setValue('way', e);
setRefresh(!refresh);
}}
></LeftRadio>
</Flex>
{showPreviewChunks && (
<Flex mt={5} alignItems={'center'} pl={'100px'} gap={3}>
<Tag colorSchema={'gray'} py={'6px'} borderRadius={'md'} px={3}>
{t('core.dataset.Total chunks', { total: totalChunks })}
</Tag>
<Tag colorSchema={'gray'} py={'6px'} borderRadius={'md'} px={3}>
{t('core.Total chars', { total: totalChunkChars })}
</Tag>
{feConfigs?.show_pay && (
<MyTooltip label={priceTip}>
<Tag colorSchema={'gray'} py={'6px'} borderRadius={'md'} px={3}>
{t('core.dataset.import.Estimated Price', { amount: predictPrice, unit: '元' })}
</Tag>
</MyTooltip>
)}
</Flex>
)}
<Flex mt={5} gap={3} justifyContent={'flex-end'}>
{showPreviewChunks && showRePreview && (
<Button variant={'primaryOutline'} onClick={splitSources2Chunks}>
{t('core.dataset.import.Re Preview')}
</Button>
)}
<Button
onClick={() => {
if (showRePreview) {
splitSources2Chunks();
}
goToNext();
}}
>
{t('common.Next Step')}
</Button>
</Flex>
</Box>
<Preview sources={sources} showPreviewChunks={showPreviewChunks} />
{isOpenCustomPrompt && (
<PromptTextarea
defaultValue={getValues('qaPrompt')}
onChange={(e) => {
setValue('qaPrompt', e);
setRefresh(!refresh);
}}
onClose={onCloseCustomPrompt}
/>
)}
</Box>
);
}
export default React.memo(DataProcess);
const PromptTextarea = ({
defaultValue,
onChange,
onClose
}: {
defaultValue: string;
onChange: (e: string) => void;
onClose: () => void;
}) => {
const ref = useRef<HTMLTextAreaElement>(null);
const { t } = useTranslation();
return (
<MyModal
isOpen
title={t('core.dataset.import.Custom prompt')}
iconSrc="modal/edit"
w={'600px'}
onClose={onClose}
>
<ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
<Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
<Box>{Prompt_AgentQA.fixedText}</Box>
</ModalBody>
<ModalFooter>
<Button
onClick={() => {
const val = ref.current?.value || Prompt_AgentQA.description;
onChange(val);
onClose();
}}
>
{t('common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};

30
projects/app/src/pages/dataset/detail/components/Import/commonProgress/PreviewData.tsx Normal file
View File

@@ -0,0 +1,30 @@
import React from 'react';
import { useImportStore } from '../Provider';
import Preview from '../components/Preview';
import { Box, Button, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
const PreviewData = ({
showPreviewChunks,
goToNext
}: {
showPreviewChunks: boolean;
goToNext: () => void;
}) => {
const { t } = useTranslation();
const { sources, setSources } = useImportStore();
console.log(sources);
return (
<Flex flexDirection={'column'} h={'100%'} maxW={'1080px'}>
<Box flex={'1 0 0 '}>
<Preview showPreviewChunks={showPreviewChunks} sources={sources} />
</Box>
<Flex mt={2} justifyContent={'flex-end'}>
<Button onClick={goToNext}>{t('common.Next Step')}</Button>
</Flex>
</Flex>
);
};
export default React.memo(PreviewData);

298
projects/app/src/pages/dataset/detail/components/Import/commonProgress/Upload.tsx Normal file
View File

@@ -0,0 +1,298 @@
import React, { useEffect, useState } from 'react';
import {
Box,
TableContainer,
Table,
Thead,
Tr,
Th,
Td,
Tbody,
Progress,
Flex,
Button
} from '@chakra-ui/react';
import { useImportStore, type FormType } from '../Provider';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRequest } from '@/web/common/hooks/useRequest';
import { postCreateTrainingBill } from '@/web/support/wallet/bill/api';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
import { chunksUpload, fileCollectionCreate } from '@/web/core/dataset/utils';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import { hashStr } from '@fastgpt/global/common/string/tools';
import { useToast } from '@/web/common/hooks/useToast';
import { useRouter } from 'next/router';
import { TabEnum } from '../../../index';
import { postCreateDatasetLinkCollection, postDatasetCollection } from '@/web/core/dataset/api';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { checkTeamDatasetSizeLimit } from '@/web/support/user/team/api';
const Upload = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => {
const { t } = useTranslation();
const { toast } = useToast();
const router = useRouter();
const { datasetDetail } = useDatasetStore();
const { parentId, sources, processParamsForm, chunkSize, totalChunks, uploadRate } =
useImportStore();
const [uploadList, setUploadList] = useState<
(ImportSourceItemType & {
uploadedFileRate: number;
uploadedChunksRate: number;
})[]
>([]);
const { handleSubmit } = processParamsForm;
const { mutate: startUpload, isLoading } = useRequest({
mutationFn: async ({ mode, customSplitChar, qaPrompt, webSelector }: FormType) => {
if (uploadList.length === 0) return;
await checkTeamDatasetSizeLimit(totalChunks);
let totalInsertion = 0;
// Batch create collection and upload chunks
for await (const item of uploadList) {
const billId = await postCreateTrainingBill({
name: item.sourceName,
datasetId: datasetDetail._id
});
// create collection
const collectionId = await (async () => {
const commonParams = {
parentId,
trainingType: mode,
datasetId: datasetDetail._id,
chunkSize,
chunkSplitter: customSplitChar,
qaPrompt,
name: item.sourceName,
rawTextLength: item.rawText.length,
hashRawText: hashStr(item.rawText)
};
if (item.file) {
return fileCollectionCreate({
file: item.file,
data: {
...commonParams,
collectionMetadata: {
relatedImgId: item.id
}
},
percentListen: (e) => {
setUploadList((state) =>
state.map((uploadItem) =>
uploadItem.id === item.id
? {
...uploadItem,
uploadedFileRate: e
}
: uploadItem
)
);
}
});
} else if (item.link) {
const { collectionId } = await postCreateDatasetLinkCollection({
...commonParams,
link: item.link,
metadata: {
webPageSelector: webSelector
}
});
setUploadList((state) =>
state.map((uploadItem) =>
uploadItem.id === item.id
? {
...uploadItem,
uploadedFileRate: 100
}
: uploadItem
)
);
return collectionId;
} else if (item.rawText) {
// manual collection
return postDatasetCollection({
...commonParams,
type: DatasetCollectionTypeEnum.virtual
});
}
return '';
})();
if (!collectionId) continue;
// upload chunks
const chunks = item.chunks;
const { insertLen } = await chunksUpload({
collectionId,
billId,
trainingMode: mode,
chunks,
rate: uploadRate,
onUploading: (e) => {
setUploadList((state) =>
state.map((uploadItem) =>
uploadItem.id === item.id
? {
...uploadItem,
uploadedChunksRate: e
}
: uploadItem
)
);
},
prompt: qaPrompt
});
totalInsertion += insertLen;
}
return totalInsertion;
},
onSuccess(num) {
if (showPreviewChunks) {
toast({
title: t('core.dataset.import.Import Success Tip', { num }),
status: 'success'
});
} else {
toast({
title: t('core.dataset.import.Upload success'),
status: 'success'
});
}
// close import page
router.replace({
query: {
...router.query,
currentTab: TabEnum.collectionCard
}
});
},
errorToast: t('common.file.Upload failed')
});
useEffect(() => {
setUploadList(
sources.map((item) => {
return {
...item,
uploadedFileRate: item.file ? 0 : -1,
uploadedChunksRate: 0
};
})
);
}, []);
return (
<Box>
<TableContainer>
<Table variant={'simple'} fontSize={'sm'} draggable={false}>
<Thead draggable={false}>
<Tr bg={'myGray.100'} mb={2}>
<Th borderLeftRadius={'md'} overflow={'hidden'} borderBottom={'none'} py={4}>
{t('core.dataset.import.Source name')}
</Th>
{showPreviewChunks ? (
<>
<Th borderBottom={'none'} py={4}>
{t('core.dataset.Chunk amount')}
</Th>
<Th borderBottom={'none'} py={4}>
{t('core.dataset.import.Upload file progress')}
</Th>
<Th borderRightRadius={'md'} overflow={'hidden'} borderBottom={'none'} py={4}>
{t('core.dataset.import.Data file progress')}
</Th>
</>
) : (
<>
<Th borderBottom={'none'} py={4}>
{t('core.dataset.import.Upload status')}
</Th>
</>
)}
</Tr>
</Thead>
<Tbody>
{uploadList.map((item) => (
<Tr key={item.id}>
<Td display={'flex'} alignItems={'center'}>
<MyIcon name={item.icon as any} w={'16px'} mr={1} />
{item.sourceName}
</Td>
{showPreviewChunks ? (
<>
<Td>{item.chunks.length}</Td>
<Td>
{item.uploadedFileRate === -1 ? (
'-'
) : (
<Flex alignItems={'center'} fontSize={'xs'}>
<Progress
value={item.uploadedFileRate}
h={'6px'}
w={'100%'}
maxW={'210px'}
size="sm"
borderRadius={'20px'}
colorScheme={'blue'}
bg="myGray.200"
hasStripe
isAnimated
mr={2}
/>
{`${item.uploadedFileRate}%`}
</Flex>
)}
</Td>
<Td>
<Flex alignItems={'center'} fontSize={'xs'}>
<Progress
value={item.uploadedChunksRate}
h={'6px'}
w={'100%'}
maxW={'210px'}
size="sm"
borderRadius={'20px'}
colorScheme={'purple'}
bg="myGray.200"
hasStripe
isAnimated
mr={2}
/>
{`${item.uploadedChunksRate}%`}
</Flex>
</Td>
</>
) : (
<>
<Td color={item.uploadedFileRate === 100 ? 'green.600' : 'myGray.600'}>
{item.uploadedFileRate === 100 ? t('common.Finish') : t('common.Waiting')}
</Td>
</>
)}
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
<Flex justifyContent={'flex-end'} mt={4}>
<Button isLoading={isLoading} onClick={handleSubmit((data) => startUpload(data))}>
{uploadList.length > 0
? `${t('core.dataset.import.Total files', { total: uploadList.length })} | `
: ''}
{t('core.dataset.import.Start upload')}
</Button>
</Flex>
</Box>
);
};
export default Upload;

134
projects/app/src/pages/dataset/detail/components/Import/components/Preview.tsx Normal file
View File

@@ -0,0 +1,134 @@
import React, { useMemo, useState } from 'react';
import { Box, Flex } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import RowTabs from '@fastgpt/web/components/common/Tabs/RowTabs';
import { ImportSourceItemType } from '@/web/core/dataset/type';
enum PreviewListEnum {
chunks = 'chunks',
sources = 'sources'
}
const Preview = ({
sources,
showPreviewChunks
}: {
sources: ImportSourceItemType[];
showPreviewChunks: boolean;
}) => {
const { t } = useTranslation();
const [previewListType, setPreviewListType] = useState(
showPreviewChunks ? PreviewListEnum.chunks : PreviewListEnum.sources
);
const chunks = useMemo(() => {
const oneSourceChunkLength = Math.max(4, Math.floor(50 / sources.length));
return sources
.map((source) =>
source.chunks.slice(0, oneSourceChunkLength).map((chunk, i) => ({
...chunk,
chunkIndex: i + 1,
sourceName: source.sourceName,
sourceIcon: source.icon
}))
)
.flat();
}, [sources]);
return (
<Box h={'100%'} display={['block', 'flex']} flexDirection={'column'} flex={'1 0 0'}>
<Box>
<RowTabs
list={[
...(showPreviewChunks
? [
{
icon: 'common/viewLight',
label: t('core.dataset.import.Preview chunks'),
value: PreviewListEnum.chunks
}
]
: []),
{
icon: 'core/dataset/fileCollection',
label: t('core.dataset.import.Sources list'),
value: PreviewListEnum.sources
}
]}
value={previewListType}
onChange={(e) => setPreviewListType(e as PreviewListEnum)}
/>
</Box>
<Box mt={3} flex={'1 0 0'} overflow={'auto'}>
{previewListType === PreviewListEnum.chunks ? (
<>
{chunks.map((chunk, i) => (
<Box
key={i}
p={4}
bg={'white'}
mb={3}
borderRadius={'md'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
boxShadow={'2'}
whiteSpace={'pre-wrap'}
>
<Flex mb={1} alignItems={'center'} fontSize={'sm'}>
<Box
flexShrink={0}
px={1}
color={'primary.600'}
borderWidth={'1px'}
borderColor={'primary.200'}
bg={'primary.50'}
borderRadius={'sm'}
>
# {chunk.chunkIndex}
</Box>
<Flex ml={2} fontWeight={'bold'} alignItems={'center'} gap={1}>
<MyIcon name={chunk.sourceIcon as any} w={'14px'} />
{chunk.sourceName}
</Flex>
</Flex>
<Box fontSize={'xs'} whiteSpace={'pre-wrap'} wordBreak={'break-all'}>
<Box color={'myGray.900'}>{chunk.q}</Box>
<Box color={'myGray.500'}>{chunk.a}</Box>
</Box>
</Box>
))}
</>
) : (
<>
{sources.map((source) => (
<Flex
key={source.id}
bg={'white'}
p={4}
borderRadius={'md'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
boxShadow={'2'}
mb={3}
>
<MyIcon name={source.icon as any} w={'16px'} />
<Box mx={1} flex={'1 0 0'} className="textEllipsis">
{source.sourceName}
</Box>
{showPreviewChunks && (
<Box>
{t('core.dataset.import.File chunk amount', { amount: source.chunks.length })}
</Box>
)}
</Flex>
))}
</>
)}
</Box>
</Box>
);
};
export default React.memo(Preview);

28
projects/app/src/pages/dataset/detail/components/Import/components/PreviewRawText.tsx Normal file
View File

@@ -0,0 +1,28 @@
import React from 'react';
import MyModal from '@/components/MyModal';
import { ModalBody } from '@chakra-ui/react';
export type PreviewRawTextProps = {
icon: string;
title: string;
rawText: string;
};
const PreviewRawText = ({
icon,
title,
rawText,
onClose
}: PreviewRawTextProps & {
onClose: () => void;
}) => {
return (
<MyModal isOpen onClose={onClose} iconSrc={icon} title={title}>
<ModalBody whiteSpace={'pre-wrap'} overflowY={'auto'}>
{rawText}
</ModalBody>
</MyModal>
);
};
export default PreviewRawText;

103
projects/app/src/pages/dataset/detail/components/Import/diffSource/FileCustomText.tsx Normal file
View File

@@ -0,0 +1,103 @@
import React, { useEffect } from 'react';
import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
import dynamic from 'next/dynamic';
import { useImportStore } from '../Provider';
import { useTranslation } from 'next-i18next';
import { useForm } from 'react-hook-form';
import { Box, Button, Flex, Input, Textarea } from '@chakra-ui/react';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import Loading from '@/components/Loading';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const CustomTet = ({ activeStep, goToNext }: ImportDataComponentProps) => {
return (
<>
{activeStep === 0 && <CustomTextInput goToNext={goToNext} />}
{activeStep === 1 && <DataProcess showPreviewChunks goToNext={goToNext} />}
{activeStep === 2 && <Upload showPreviewChunks />}
</>
);
};
export default React.memo(CustomTet);
const CustomTextInput = ({ goToNext }: { goToNext: () => void }) => {
const { t } = useTranslation();
const { sources, setSources } = useImportStore();
const { register, reset, handleSubmit } = useForm({
defaultValues: {
name: '',
value: ''
}
});
useEffect(() => {
const source = sources[0];
if (source) {
reset({
name: source.sourceName,
value: source.rawText
});
}
}, []);
return (
<Box maxW={['100%', '800px']}>
<Box display={['block', 'flex']} alignItems={'center'}>
<Box flex={'0 0 120px'} fontSize={'sm'}>
{t('core.dataset.collection.Collection name')}
</Box>
<Input
flex={'1 0 0'}
maxW={['100%', '350px']}
{...register('name', {
required: true
})}
placeholder={t('core.dataset.collection.Collection name')}
bg={'myGray.50'}
/>
</Box>
<Box display={['block', 'flex']} alignItems={'flex-start'} mt={5}>
<Box flex={'0 0 120px'} fontSize={'sm'}>
{t('core.dataset.collection.Collection raw text')}
</Box>
<Textarea
flex={'1 0 0'}
w={'100%'}
rows={15}
placeholder={t('core.dataset.collection.Collection raw text')}
{...register('value', {
required: true
})}
bg={'myGray.50'}
/>
</Box>
<Flex mt={5} justifyContent={'flex-end'}>
<Button
onClick={handleSubmit((data) => {
const fileId = getNanoid(32);
setSources([
{
id: fileId,
rawText: data.value,
chunks: [],
chunkChars: 0,
sourceName: data.name,
icon: 'file/fill/manual'
}
]);
goToNext();
})}
>
{t('common.Next Step')}
</Button>
</Flex>
</Box>
);
};

147
projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLink.tsx Normal file
View File

@@ -0,0 +1,147 @@
import React, { useEffect } from 'react';
import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
import dynamic from 'next/dynamic';
import { useImportStore } from '../Provider';
import { useTranslation } from 'next-i18next';
import { useForm } from 'react-hook-form';
import { Box, Button, Flex, Input, Link, Textarea } from '@chakra-ui/react';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { LinkCollectionIcon } from '@fastgpt/global/core/dataset/constants';
import { feConfigs } from '@/web/common/system/staticData';
import { getDocPath } from '@/web/common/system/doc';
import Loading from '@/components/Loading';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const LinkCollection = ({ activeStep, goToNext }: ImportDataComponentProps) => {
return (
<>
{activeStep === 0 && <CustomLinkImport goToNext={goToNext} />}
{activeStep === 1 && <DataProcess showPreviewChunks={false} goToNext={goToNext} />}
{activeStep === 2 && <Upload showPreviewChunks={false} />}
</>
);
};
export default React.memo(LinkCollection);
const CustomLinkImport = ({ goToNext }: { goToNext: () => void }) => {
const { t } = useTranslation();
const { sources, setSources, processParamsForm } = useImportStore();
const { register, reset, handleSubmit, watch } = useForm({
defaultValues: {
link: ''
}
});
const link = watch('link');
const linkList = link.split('\n').filter((item) => item);
useEffect(() => {
reset({
link: sources
.map((item) => item.link)
.filter((item) => item)
.join('\n')
});
}, []);
return (
<Box maxW={['100%', '800px']}>
<Box display={['block', 'flex']} alignItems={'flex-start'} mt={1}>
<Box flex={'0 0 100px'} fontSize={'sm'}>
{t('core.dataset.import.Link name')}
</Box>
<Textarea
flex={'1 0 0'}
w={'100%'}
rows={10}
placeholder={t('core.dataset.import.Link name placeholder')}
bg={'myGray.50'}
overflowX={'auto'}
whiteSpace={'nowrap'}
{...register('link', {
required: true
})}
/>
</Box>
<Box display={['block', 'flex']} alignItems={'center'} mt={4}>
<Box flex={'0 0 100px'} fontSize={'sm'}>
{t('core.dataset.website.Selector')}
<Box color={'myGray.500'} fontSize={'sm'}>
{feConfigs?.docUrl && (
<Link href={getDocPath('/docs/course/websync/#选择器如何使用')} target="_blank">
{t('core.dataset.website.Selector Course')}
</Link>
)}
</Box>
</Box>
<Input
flex={'1 0 0'}
maxW={['100%', '350px']}
{...processParamsForm.register('webSelector')}
placeholder={'body .content #document'}
bg={'myGray.50'}
/>
</Box>
<Flex my={4} flexWrap={'wrap'} gap={4} alignItems={'center'} pl={[0, '100px']}>
{linkList.map((item, i) => (
<Flex
key={`${item}-${i}`}
alignItems={'center'}
px={4}
py={2}
borderRadius={'md'}
bg={'myGray.100'}
>
<MyIcon name={LinkCollectionIcon} w={'16px'} />
<Box ml={1} mr={3} wordBreak={'break-all'}>
{item}
</Box>
<MyIcon
name={'common/closeLight'}
w={'14px'}
color={'myGray.500'}
cursor={'pointer'}
onClick={() => {
const newLinkList = linkList.filter((link, index) => index !== i);
reset({
link: newLinkList.join('\n')
});
}}
/>
</Flex>
))}
</Flex>
<Flex mt={5} justifyContent={'flex-end'}>
<Button
onClick={handleSubmit((data) => {
const newLinkList = data.link.split('\n').filter((item) => item);
setSources(
newLinkList.map((link) => ({
id: getNanoid(32),
link,
rawText: '',
chunks: [],
chunkChars: 0,
sourceName: link,
icon: LinkCollectionIcon
}))
);
goToNext();
})}
>
{t('common.Next Step')}
</Button>
</Flex>
</Box>
);
};

177
projects/app/src/pages/dataset/detail/components/Import/diffSource/FileLocal.tsx Normal file
View File

@@ -0,0 +1,177 @@
import React, { useEffect, useMemo, useState } from 'react';
import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
import { Box, Button, Flex } from '@chakra-ui/react';
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
import FileSelector, { type SelectFileItemType } from '@/web/core/dataset/components/FileSelector';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { useTranslation } from 'next-i18next';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { useRequest } from '@/web/common/hooks/useRequest';
import { readFileRawContent } from '@fastgpt/web/common/file/read';
import { getUploadBase64ImgController } from '@/web/common/file/controller';
import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
import MyTooltip from '@/components/MyTooltip';
import type { PreviewRawTextProps } from '../components/PreviewRawText';
import { useImportStore } from '../Provider';
import { feConfigs } from '@/web/common/system/staticData';
import dynamic from 'next/dynamic';
import Loading from '@/components/Loading';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const PreviewRawText = dynamic(() => import('../components/PreviewRawText'));
type FileItemType = ImportSourceItemType & { file: File };
const fileType = '.txt, .docx, .pdf, .md, .html';
const maxSelectFileCount = 1000;
const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => {
return (
<>
{activeStep === 0 && <SelectFile goToNext={goToNext} />}
{activeStep === 1 && <DataProcess showPreviewChunks goToNext={goToNext} />}
{activeStep === 2 && <Upload showPreviewChunks />}
</>
);
};
export default React.memo(FileLocal);
const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) {
const { t } = useTranslation();
const { sources, setSources } = useImportStore();
// @ts-ignore
const [selectFiles, setSelectFiles] = useState<FileItemType[]>(sources);
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
const [previewRaw, setPreviewRaw] = useState<PreviewRawTextProps>();
useEffect(() => {
setSources(successFiles);
}, [successFiles]);
const { mutate: onSelectFile, isLoading } = useRequest({
mutationFn: async (files: SelectFileItemType[]) => {
{
for await (const selectFile of files) {
const { file, folderPath } = selectFile;
const relatedId = getNanoid(32);
const { rawText } = await (() => {
try {
return readFileRawContent({
file,
uploadBase64Controller: (base64Img) =>
getUploadBase64ImgController({
base64Img,
type: MongoImageTypeEnum.collectionImage,
metadata: {
relatedId
}
})
});
} catch (error) {
return { rawText: '' };
}
})();
const item: FileItemType = {
id: relatedId,
file,
rawText,
chunks: [],
chunkChars: 0,
sourceFolderPath: folderPath,
sourceName: file.name,
sourceSize: formatFileSize(file.size),
icon: getFileIcon(file.name),
errorMsg: rawText.length === 0 ? t('common.file.Empty file tip') : ''
};
setSelectFiles((state) => {
const results = [item].concat(state).slice(0, maxSelectFileCount);
return results;
});
}
}
}
});
return (
<Box>
<FileSelector
isLoading={isLoading}
fileType={fileType}
multiple
maxCount={maxSelectFileCount}
maxSize={(feConfigs?.uploadFileMaxSize || 500) * 1024 * 1024}
onSelectFile={onSelectFile}
/>
{/* render files */}
<Flex my={4} flexWrap={'wrap'} gap={5} alignItems={'center'}>
{selectFiles.map((item) => (
<MyTooltip key={item.id} label={t('core.dataset.import.Preview raw text')}>
<Flex
alignItems={'center'}
px={4}
py={3}
borderRadius={'md'}
bg={'myGray.100'}
cursor={'pointer'}
onClick={() =>
setPreviewRaw({
icon: item.icon,
title: item.sourceName,
rawText: item.rawText.slice(0, 10000)
})
}
>
<MyIcon name={item.icon as any} w={'16px'} />
<Box ml={1} mr={3}>
{item.sourceName}
</Box>
<Box mr={1} fontSize={'xs'} color={'myGray.500'}>
{item.sourceSize}
{item.rawText.length > 0 && (
<>,{t('common.Number of words', { amount: item.rawText.length })}</>
)}
</Box>
{item.errorMsg && (
<MyTooltip label={item.errorMsg}>
<MyIcon name={'common/errorFill'} w={'14px'} mr={3} />
</MyTooltip>
)}
<MyIcon
name={'common/closeLight'}
w={'14px'}
color={'myGray.500'}
cursor={'pointer'}
onClick={(e) => {
e.stopPropagation();
setSelectFiles((state) => state.filter((file) => file.id !== item.id));
}}
/>
</Flex>
</MyTooltip>
))}
</Flex>
<Box textAlign={'right'}>
<Button isDisabled={successFiles.length === 0 || isLoading} onClick={goToNext}>
{selectFiles.length > 0
? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
: ''}
{t('common.Next Step')}
</Button>
</Box>
{previewRaw && <PreviewRawText {...previewRaw} onClose={() => setPreviewRaw(undefined)} />}
</Box>
);
});

168
projects/app/src/pages/dataset/detail/components/Import/diffSource/TableLocal.tsx Normal file
View File

@@ -0,0 +1,168 @@
import React, { useEffect, useMemo, useState } from 'react';
import { ImportDataComponentProps } from '@/web/core/dataset/type.d';
import { Box, Button, Flex } from '@chakra-ui/react';
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
import FileSelector, { type SelectFileItemType } from '@/web/core/dataset/components/FileSelector';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { useTranslation } from 'next-i18next';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { useRequest } from '@/web/common/hooks/useRequest';
import MyTooltip from '@/components/MyTooltip';
import { useImportStore } from '../Provider';
import { feConfigs } from '@/web/common/system/staticData';
import dynamic from 'next/dynamic';
import { fileDownload, readCsvContent } from '@/web/common/file/utils';
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
const Upload = dynamic(() => import('../commonProgress/Upload'));
type FileItemType = ImportSourceItemType & { file: File };
const fileType = '.csv';
const maxSelectFileCount = 1000;
const FileLocal = ({ activeStep, goToNext }: ImportDataComponentProps) => {
return (
<>
{activeStep === 0 && <SelectFile goToNext={goToNext} />}
{activeStep === 1 && <PreviewData showPreviewChunks goToNext={goToNext} />}
{activeStep === 2 && <Upload showPreviewChunks />}
</>
);
};
export default React.memo(FileLocal);
const csvTemplate = `index,content
"必填内容","可选内容。CSV 中请注意内容不能包含双引号，双引号是列分割符号"
"结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。",""
"AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`;
const SelectFile = React.memo(function SelectFile({ goToNext }: { goToNext: () => void }) {
const { t } = useTranslation();
const { sources, setSources } = useImportStore();
// @ts-ignore
const [selectFiles, setSelectFiles] = useState<FileItemType[]>(sources);
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
useEffect(() => {
setSources(successFiles);
}, [successFiles]);
const { mutate: onSelectFile, isLoading } = useRequest({
mutationFn: async (files: SelectFileItemType[]) => {
{
for await (const selectFile of files) {
const { file, folderPath } = selectFile;
const { header, data } = await readCsvContent(file);
const filterData: FileItemType['chunks'] = data
.filter((item) => item[0])
.map((item, i) => ({
q: item[0] || '',
a: item[1] || '',
chunkIndex: i
}));
const item: FileItemType = {
id: getNanoid(32),
file,
rawText: '',
chunks: filterData,
chunkChars: 0,
sourceFolderPath: folderPath,
sourceName: file.name,
sourceSize: formatFileSize(file.size),
icon: getFileIcon(file.name),
errorMsg:
header[0] !== 'index' || header[1] !== 'content' || filterData.length === 0
? t('core.dataset.import.Csv format error')
: ''
};
setSelectFiles((state) => {
const results = [item].concat(state).slice(0, 10);
return results;
});
}
}
},
errorToast: t('common.file.Select failed')
});
return (
<Box>
<FileSelector
multiple
maxCount={maxSelectFileCount}
maxSize={(feConfigs?.uploadFileMaxSize || 500) * 1024 * 1024}
isLoading={isLoading}
fileType={fileType}
onSelectFile={onSelectFile}
/>
<Box
mt={4}
color={'primary.600'}
textDecoration={'underline'}
cursor={'pointer'}
onClick={() =>
fileDownload({
text: csvTemplate,
type: 'text/csv;charset=utf-8',
filename: 'template.csv'
})
}
>
{t('core.dataset.import.Down load csv template')}
</Box>
{/* render files */}
<Flex my={4} flexWrap={'wrap'} gap={5} alignItems={'center'}>
{selectFiles.map((item) => (
<Flex
key={item.id}
alignItems={'center'}
px={4}
py={2}
borderRadius={'md'}
bg={'myGray.100'}
>
<MyIcon name={item.icon as any} w={'16px'} />
<Box ml={1} mr={3}>
{item.sourceName}
</Box>
<Box mr={1} fontSize={'xs'} color={'myGray.500'}>
{item.sourceSize}
</Box>
{item.errorMsg && (
<MyTooltip label={item.errorMsg}>
<MyIcon name={'common/errorFill'} w={'14px'} mr={3} />
</MyTooltip>
)}
<MyIcon
name={'common/closeLight'}
w={'14px'}
color={'myGray.500'}
cursor={'pointer'}
onClick={() => {
setSelectFiles((state) => state.filter((file) => file.id !== item.id));
}}
/>
</Flex>
))}
</Flex>
<Box textAlign={'right'}>
<Button isDisabled={successFiles.length === 0 || isLoading} onClick={goToNext}>
{selectFiles.length > 0
? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
: ''}
{t('common.Next Step')}
</Button>
</Box>
</Box>
);
});

154
projects/app/src/pages/dataset/detail/components/Import/index.tsx Normal file
View File

@@ -0,0 +1,154 @@
import React, { useMemo } from 'react';
import { Box, Button, Flex, IconButton } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import { useRouter } from 'next/router';
import { TabEnum } from '../../index';
import { useMyStep } from '@fastgpt/web/hooks/useStep';
import dynamic from 'next/dynamic';
import Provider from './Provider';
import { useDatasetStore } from '@/web/core/dataset/store/dataset';
const FileLocal = dynamic(() => import('./diffSource/FileLocal'));
const FileLink = dynamic(() => import('./diffSource/FileLink'));
const FileCustomText = dynamic(() => import('./diffSource/FileCustomText'));
const TableLocal = dynamic(() => import('./diffSource/TableLocal'));
export enum ImportDataSourceEnum {
fileLocal = 'fileLocal',
fileLink = 'fileLink',
fileCustom = 'fileCustom',
tableLocal = 'tableLocal'
}
const ImportDataset = () => {
const { t } = useTranslation();
const router = useRouter();
const { datasetDetail } = useDatasetStore();
const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as {
source: `${ImportDataSourceEnum}`;
parentId?: string;
};
const modeSteps: Record<`${ImportDataSourceEnum}`, { title: string }[]> = {
[ImportDataSourceEnum.fileLocal]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.fileLink]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.fileCustom]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.tableLocal]: [
{
title: t('core.dataset.import.Select file')
},
{
title: t('core.dataset.import.Data Preprocessing')
},
{
title: t('core.dataset.import.Upload data')
}
]
};
const steps = modeSteps[source];
const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({
defaultStep: 0,
steps
});
const ImportComponent = useMemo(() => {
if (source === ImportDataSourceEnum.fileLocal) return FileLocal;
if (source === ImportDataSourceEnum.fileLink) return FileLink;
if (source === ImportDataSourceEnum.fileCustom) return FileCustomText;
if (source === ImportDataSourceEnum.tableLocal) return TableLocal;
}, [source]);
return ImportComponent ? (
<Flex flexDirection={'column'} bg={'white'} h={'100%'} px={[2, 9]} py={[2, 5]}>
<Flex>
{activeStep === 0 ? (
<Flex alignItems={'center'}>
<IconButton
icon={<MyIcon name={'common/backFill'} w={'14px'} />}
aria-label={''}
size={'smSquare'}
w={'26px'}
h={'26px'}
borderRadius={'50%'}
variant={'whiteBase'}
mr={2}
onClick={() =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.collectionCard
}
})
}
/>
{t('common.Exit')}
</Flex>
) : (
<Button
variant={'whiteBase'}
leftIcon={<MyIcon name={'common/backFill'} w={'14px'} />}
onClick={goToPrevious}
>
{t('common.Last Step')}
</Button>
)}
<Box flex={1} />
</Flex>
{/* step */}
<Box
mt={4}
mb={5}
px={3}
py={[2, 4]}
bg={'myGray.50'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
borderRadius={'md'}
>
<Box maxW={['100%', '900px']} mx={'auto'}>
<MyStep />
</Box>
</Box>
<Provider dataset={datasetDetail} parentId={parentId}>
<Box flex={'1 0 0'} overflow={'auto'} position={'relative'}>
<ImportComponent activeStep={activeStep} goToNext={goToNext} />
</Box>
</Provider>
</Flex>
) : null;
};
export default React.memo(ImportDataset);

65
projects/app/src/pages/dataset/detail/components/Import/sourceSelector/FileSourceSelector.tsx Normal file
View File

@@ -0,0 +1,65 @@
import React, { useState } from 'react';
import MyModal from '@/components/MyModal';
import { ModalBody, ModalFooter, Button } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import { ImportDataSourceEnum } from '..';
import { useRouter } from 'next/router';
import { TabEnum } from '../../..';
const FileModeSelector = ({ onClose }: { onClose: () => void }) => {
const { t } = useTranslation();
const router = useRouter();
const [value, setValue] = useState<`${ImportDataSourceEnum}`>(ImportDataSourceEnum.fileLocal);
return (
<MyModal
isOpen
onClose={onClose}
iconSrc="modal/selectSource"
title={t('core.dataset.import.Select source')}
w={'600px'}
>
<ModalBody px={6} py={4}>
<LeftRadio
list={[
{
title: t('core.dataset.import.Local file'),
desc: t('core.dataset.import.Local file desc'),
value: ImportDataSourceEnum.fileLocal
},
{
title: t('core.dataset.import.Web link'),
desc: t('core.dataset.import.Web link desc'),
value: ImportDataSourceEnum.fileLink
},
{
title: t('core.dataset.import.Custom text'),
desc: t('core.dataset.import.Custom text desc'),
value: ImportDataSourceEnum.fileCustom
}
]}
value={value}
onChange={setValue}
/>
</ModalBody>
<ModalFooter>
<Button
onClick={() =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.import,
source: value
}
})
}
>
{t('common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};
export default FileModeSelector;

61
projects/app/src/pages/dataset/detail/components/Info.tsx
View File

@@ -14,7 +14,7 @@ import MyTooltip from '@/components/MyTooltip';
import { useTranslation } from 'next-i18next';
import PermissionRadio from '@/components/support/permission/Radio';
import MySelect from '@/components/Select';
import { qaModelList } from '@/web/common/system/staticData';
import { qaModelList, vectorModelList } from '@/web/common/system/staticData';
import { useRequest } from '@/web/common/hooks/useRequest';
import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
@@ -119,39 +119,44 @@ const Info = ({ datasetId }: { datasetId: string }) => {
</Box>
<Input flex={[1, '0 0 300px']} maxLength={30} {...register('name')} />
</Flex>
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.ai.model.Vector Model')}
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
</Flex>
{vectorModelList.length > 1 && (
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.ai.model.Vector Model')}
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').name}</Box>
</Flex>
)}
<Flex mt={8} w={'100%'} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.Max Token')}
</Box>
<Box flex={[1, '0 0 300px']}>{getValues('vectorModel').maxToken}</Box>
</Flex>
<Flex mt={6} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.ai.model.Dataset Agent Model')}
</Box>
<Box flex={[1, '0 0 300px']}>
<MySelect
w={'100%'}
value={getValues('agentModel').model}
list={qaModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
const agentModel = qaModelList.find((item) => item.model === e);
if (!agentModel) return;
setValue('agentModel', agentModel);
setRefresh((state) => !state);
}}
/>
</Box>
</Flex>
{qaModelList.length > 1 && (
<Flex mt={6} alignItems={'center'}>
<Box flex={['0 0 90px', '0 0 160px']} w={0}>
{t('core.ai.model.Dataset Agent Model')}
</Box>
<Box flex={[1, '0 0 300px']}>
<MySelect
w={'100%'}
value={getValues('agentModel').model}
list={qaModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
const agentModel = qaModelList.find((item) => item.model === e);
if (!agentModel) return;
setValue('agentModel', agentModel);
setRefresh((state) => !state);
}}
/>
</Box>
</Flex>
)}
<Flex mt={8} alignItems={'center'} w={'100%'}>
<Box flex={['0 0 90px', '0 0 160px']}>{t('common.Intro')}</Box>
<Textarea flex={[1, '0 0 300px']} {...register('intro')} placeholder={t('common.Intro')} />

132
projects/app/src/pages/dataset/detail/components/InputDataModal.tsx
View File

@@ -1,6 +1,6 @@
import React, { useMemo, useState } from 'react';
import { Box, Flex, Button, Textarea, useTheme, Grid } from '@chakra-ui/react';
import { useFieldArray, useForm } from 'react-hook-form';
import { UseFormRegister, useFieldArray, useForm } from 'react-hook-form';
import {
postInsertData2Dataset,
putDatasetDataById,
@@ -20,7 +20,7 @@ import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils';
import { vectorModelList } from '@/web/common/system/staticData';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
import SideTabs from '@/components/SideTabs';
import DeleteIcon from '@fastgpt/web/components/common/Icon/delete';
@@ -29,6 +29,8 @@ import { getDocPath } from '@/web/common/system/doc';
import RawSourceBox from '@/components/core/dataset/RawSourceBox';
import MyBox from '@/components/common/MyBox';
import { getErrText } from '@fastgpt/global/common/error/utils';
import RowTabs from '@fastgpt/web/components/common/Tabs/RowTabs';
import { useSystemStore } from '@/web/common/system/useSystemStore';
export type InputDataType = {
q: string;
@@ -124,7 +126,7 @@ const InputDataModal = ({
onError(err) {
toast({
status: 'error',
title: getErrText(err)
title: t(getErrText(err))
});
onClose();
}
@@ -261,51 +263,7 @@ const InputDataModal = ({
{currentTab === TabEnum.index && <> {t('dataset.data.Index Edit')}</>}
</Box>
<Box flex={1} px={5} overflow={'auto'}>
{currentTab === TabEnum.content && (
<>
<Box>
<Flex alignItems={'center'}>
<Box>
<Box as="span" color={'red.600'}>
*
</Box>
{t('core.dataset.data.Data Content')}
</Box>
<MyTooltip label={t('core.dataset.data.Data Content Tip')}>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Flex>
<Textarea
mt={1}
placeholder={t('core.dataset.data.Data Content Placeholder', { maxToken })}
maxLength={maxToken}
rows={12}
bg={'myWhite.400'}
{...register(`q`, {
required: true
})}
/>
</Box>
<Box mt={5}>
<Flex>
<Box>{t('core.dataset.data.Auxiliary Data')}</Box>
<MyTooltip label={t('core.dataset.data.Auxiliary Data Tip')}>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Flex>
<Textarea
mt={1}
placeholder={t('core.dataset.data.Auxiliary Data Placeholder', {
maxToken: maxToken * 1.5
})}
bg={'myWhite.400'}
rows={12}
maxLength={maxToken * 1.5}
{...register('a')}
/>
</Box>
</>
)}
{currentTab === TabEnum.content && <InputTab maxToken={maxToken} register={register} />}
{currentTab === TabEnum.index && (
<Grid gridTemplateColumns={['1fr', '1fr 1fr']} gridGap={4}>
{indexes.map((index, i) => (
@@ -382,6 +340,7 @@ const InputDataModal = ({
</Grid>
)}
</Box>
{/* footer */}
<Flex justifyContent={'flex-end'} px={5} mt={4}>
<Button variant={'whiteBase'} mr={3} onClick={onClose}>
{t('common.Close')}
@@ -404,3 +363,80 @@ const InputDataModal = ({
};
export default React.memo(InputDataModal);
enum InputTypeEnum {
q = 'q',
a = 'a'
}
const InputTab = ({
maxToken,
register
}: {
maxToken: number;
register: UseFormRegister<InputDataType>;
}) => {
const { t } = useTranslation();
const { isPc } = useSystemStore();
const [inputType, setInputType] = useState(InputTypeEnum.q);
return (
<Box>
<RowTabs
list={[
{
label: (
<Flex alignItems={'center'}>
<Box as="span" color={'red.600'}>
*
</Box>
{t('core.dataset.data.Main Content')}
<MyTooltip label={t('core.dataset.data.Data Content Tip')}>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Flex>
),
value: InputTypeEnum.q
},
{
label: (
<Flex alignItems={'center'}>
{t('core.dataset.data.Auxiliary Data')}
<MyTooltip label={t('core.dataset.data.Auxiliary Data Tip')}>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Flex>
),
value: InputTypeEnum.a
}
]}
value={inputType}
onChange={(e) => setInputType(e as InputTypeEnum)}
/>
<Box mt={3}>
{inputType === InputTypeEnum.q && (
<Textarea
placeholder={t('core.dataset.data.Data Content Placeholder', { maxToken })}
maxLength={maxToken}
rows={isPc ? 24 : 12}
bg={'myWhite.400'}
{...register(`q`, {
required: true
})}
/>
)}
{inputType === InputTypeEnum.a && (
<Textarea
placeholder={t('core.dataset.data.Auxiliary Data Placeholder', {
maxToken: maxToken * 1.5
})}
bg={'myWhite.400'}
rows={isPc ? 24 : 12}
maxLength={maxToken * 1.5}
{...register('a')}
/>
)}
</Box>
</Box>
);
};

15
projects/app/src/pages/dataset/detail/components/Test.tsx
View File

@@ -5,7 +5,6 @@ import {
Button,
Flex,
useTheme,
Grid,
useDisclosure,
Table,
Thead,
@@ -28,7 +27,10 @@ import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import { useTranslation } from 'next-i18next';
import { SearchTestResponse } from '@/global/core/dataset/api';
import { DatasetSearchModeEnum, DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constant';
import {
DatasetSearchModeEnum,
DatasetSearchModeMap
} from '@fastgpt/global/core/dataset/constants';
import dynamic from 'next/dynamic';
import { useForm } from 'react-hook-form';
import MySelect from '@/components/Select';
@@ -97,6 +99,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
title: t('dataset.test.noResult')
});
}
const testItem: SearchTestStoreItemType = {
id: nanoid(),
datasetId,
@@ -389,7 +392,7 @@ const TestHistories = React.memo(function TestHistories({
})}
onClick={() => setDatasetTestItem(item)}
>
<Box flex={'0 0 80px'}>
<Box flex={'0 0 auto'} mr={2}>
{DatasetSearchModeMap[item.searchMode] ? (
<Flex alignItems={'center'} fontWeight={'500'} color={'myGray.500'}>
<MyIcon
@@ -406,7 +409,11 @@ const TestHistories = React.memo(function TestHistories({
<Box flex={1} mr={2} wordBreak={'break-all'} fontWeight={'400'}>
{item.text}
</Box>
<Box flex={'0 0 70px'}>{formatTimeToChatTime(item.time)}</Box>
<Box flex={'0 0 70px'}>
{formatTimeToChatTime(item.time).includes('.')
? t(formatTimeToChatTime(item.time))
: formatTimeToChatTime(item.time)}
</Box>
<MyTooltip label={t('core.dataset.test.delete test history')}>
<Box w={'14px'} h={'14px'}>
<MyIcon

25
projects/app/src/pages/dataset/detail/index.tsx
View File

@@ -11,7 +11,6 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
import SideTabs from '@/components/SideTabs';
import PageContainer from '@/components/PageContainer';
import Avatar from '@/components/Avatar';
import Info from './components/Info';
import { serviceSideProps } from '@/web/common/utils/i18n';
import { useTranslation } from 'next-i18next';
import { getTrainingQueueLen } from '@/web/core/dataset/api';
@@ -24,23 +23,22 @@ import {
DatasetStatusEnum,
DatasetTypeEnum,
DatasetTypeMap
} from '@fastgpt/global/core/dataset/constant';
} from '@fastgpt/global/core/dataset/constants';
import { useConfirm } from '@/web/common/hooks/useConfirm';
import { useRequest } from '@/web/common/hooks/useRequest';
import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag';
const DataCard = dynamic(() => import('./components/DataCard'), {
ssr: false
});
const Test = dynamic(() => import('./components/Test'), {
ssr: false
});
const DataCard = dynamic(() => import('./components/DataCard'));
const Test = dynamic(() => import('./components/Test'));
const Info = dynamic(() => import('./components/Info'));
const Import = dynamic(() => import('./components/Import'));
export enum TabEnum {
dataCard = 'dataCard',
collectionCard = 'collectionCard',
test = 'test',
info = 'info'
info = 'info',
import = 'import'
}
const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${TabEnum}` }) => {
@@ -53,7 +51,11 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
const { userInfo } = useUserStore();
const tabList = [
{ label: t('core.dataset.Dataset'), id: TabEnum.collectionCard, icon: 'common/overviewLight' },
{
label: t('core.dataset.Collection'),
id: TabEnum.collectionCard,
icon: 'common/overviewLight'
},
{ label: t('core.dataset.test.Search Test'), id: TabEnum.test, icon: 'kbTest' },
...(userInfo?.team.canWrite && datasetDetail.isOwner
? [{ label: t('common.Config'), id: TabEnum.info, icon: 'common/settingLight' }]
@@ -264,11 +266,12 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T
)}
{!!datasetDetail._id && (
<Box flex={'1 0 0'} pb={[4, 0]}>
<Box flex={'1 0 0'} pb={0}>
{currentTab === TabEnum.collectionCard && <CollectionCard />}
{currentTab === TabEnum.dataCard && <DataCard />}
{currentTab === TabEnum.test && <Test datasetId={datasetId} />}
{currentTab === TabEnum.info && <Info datasetId={datasetId} />}
{currentTab === TabEnum.import && <Import />}
</Box>
)}
</Flex>

74
projects/app/src/pages/dataset/list/component/CreateModal.tsx
View File

@@ -17,7 +17,7 @@ import MySelect from '@/components/Select';
import { vectorModelList, qaModelList } from '@/web/common/system/staticData';
import { useTranslation } from 'next-i18next';
import MyRadio from '@/components/common/MyRadio';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { feConfigs } from '@/web/common/system/staticData';
import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants';
@@ -149,40 +149,44 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st
/>
</Flex>
</Box>
<Flex mt={6} alignItems={'center'}>
<Box flex={'0 0 100px'}>{t('core.ai.model.Vector Model')}</Box>
<Box flex={1}>
<MySelect
w={'100%'}
value={getValues('vectorModel')}
list={vectorModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
setValue('vectorModel', e);
setRefresh((state) => !state);
}}
/>
</Box>
</Flex>
<Flex mt={6} alignItems={'center'}>
<Box flex={'0 0 100px'}>{t('core.ai.model.Dataset Agent Model')}</Box>
<Box flex={1}>
<MySelect
w={'100%'}
value={getValues('agentModel')}
list={qaModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
setValue('agentModel', e);
setRefresh((state) => !state);
}}
/>
</Box>
</Flex>
{vectorModelList.length > 1 && (
<Flex mt={6} alignItems={'center'}>
<Box flex={'0 0 100px'}>{t('core.ai.model.Vector Model')}</Box>
<Box flex={1}>
<MySelect
w={'100%'}
value={getValues('vectorModel')}
list={vectorModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
setValue('vectorModel', e);
setRefresh((state) => !state);
}}
/>
</Box>
</Flex>
)}
{qaModelList.length > 1 && (
<Flex mt={6} alignItems={'center'}>
<Box flex={'0 0 100px'}>{t('core.ai.model.Dataset Agent Model')}</Box>
<Box flex={1}>
<MySelect
w={'100%'}
value={getValues('agentModel')}
list={qaModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
setValue('agentModel', e);
setRefresh((state) => !state);
}}
/>
</Box>
</Flex>
)}
</ModalBody>
<ModalFooter>

2
projects/app/src/pages/dataset/list/component/MoveModal.tsx
View File

@@ -14,7 +14,7 @@ import Avatar from '@/components/Avatar';
import MyTooltip from '@/components/MyTooltip';
import MyModal from '@/components/MyModal';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { useTranslation } from 'next-i18next';
import { useQuery } from '@tanstack/react-query';
import { getDatasets, putDatasetById, getDatasetPaths } from '@/web/core/dataset/api';

18
projects/app/src/pages/dataset/list/index.tsx
View File

@@ -29,10 +29,11 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
import { serviceSideProps } from '@/web/common/utils/i18n';
import dynamic from 'next/dynamic';
import {
FolderAvatarSrc,
DatasetTypeEnum,
DatasetTypeMap
} from '@fastgpt/global/core/dataset/constant';
DatasetTypeMap,
FolderIcon,
FolderImgUrl
} from '@fastgpt/global/core/dataset/constants';
import MyMenu from '@/components/MyMenu';
import { useRequest } from '@/web/common/hooks/useRequest';
import { useSystemStore } from '@/web/common/system/useSystemStore';
@@ -167,7 +168,7 @@ const Kb = () => {
<MenuButton h={'100%'}>
<Flex alignItems={'center'} px={'20px'}>
<AddIcon mr={2} />
<Box>{t('Create New')}</Box>
<Box>{t('common.Create New')}</Box>
</Flex>
</MenuButton>
</Button>
@@ -176,7 +177,7 @@ const Kb = () => {
{
child: (
<Flex>
<Image src={FolderAvatarSrc} alt={''} w={'20px'} mr={1} />
<MyIcon name={FolderIcon} w={'20px'} mr={1} />
{t('Folder')}
</Flex>
),
@@ -404,7 +405,10 @@ const Kb = () => {
fontSize={'sm'}
color={'myGray.500'}
>
{dataset.intro || t('core.dataset.Intro Placeholder')}
{dataset.intro ||
(dataset.type === DatasetTypeEnum.folder
? t('core.dataset.Folder placeholder')
: t('core.dataset.Intro Placeholder'))}
</Box>
<Flex alignItems={'center'} fontSize={'sm'}>
<Box flex={1}>
@@ -437,7 +441,7 @@ const Kb = () => {
parentId,
name,
type: DatasetTypeEnum.folder,
avatar: FolderAvatarSrc,
avatar: FolderImgUrl,
intro: ''
});
refetch();

2
projects/app/src/pages/login/components/LoginForm.tsx
View File

@@ -114,7 +114,7 @@ const LoginForm = ({ setPageType, loginSuccess }: Props) => {
bg={'myGray.25'}
borderRadius={'xl'}
borderWidth={'1.5px'}
borderColor={theme.borderColor.borderColor}
borderColor={'borderColor.base'}
alignItems={'center'}
justifyContent={'center'}
>

2
projects/app/src/pages/login/index.tsx
View File

@@ -77,7 +77,7 @@ const Login = () => {
flexDirection={'column'}
w={['100%', 'auto']}
h={['100%', '700px']}
maxH={'90vh'}
maxH={['100%', '90vh']}
bg={'white'}
px={['5vw', '88px']}
py={'5vh'}