gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feat: Images dataset collection (#4941)

Browse Source 
* New pic (#4858)

* 更新数据集相关类型，添加图像文件ID和预览URL支持；优化数据集导入功能，新增图像数据集处理组件；修复部分国际化文本；更新文件上传逻辑以支持新功能。

* 与原先代码的差别

* 新增 V4.9.10 更新说明，支持 PG 设置`systemEnv.hnswMaxScanTuples`参数，优化 LLM stream 调用超时，修复全文检索多知识库排序问题。同时更新数据集索引，移除 datasetId 字段以简化查询。

* 更换成fileId_image逻辑，并增加训练队列匹配的逻辑

* 新增图片集合判断逻辑，优化预览URL生成流程，确保仅在数据集为图片集合时生成预览URL，并添加相关日志输出以便调试。

* Refactor Docker Compose configuration to comment out exposed ports for production environments, update image versions for pgvector, fastgpt, and mcp_server, and enhance Redis service with a health check. Additionally, standardize dataset collection labels in constants and improve internationalization strings across multiple languages.

* Enhance TrainingStates component by adding internationalization support for the imageParse training mode and update defaultCounts to include imageParse mode in trainingDetail API.

* Enhance dataset import context by adding additional steps for image dataset import process and improve internationalization strings for modal buttons in the useEditTitle hook.

* Update DatasetImportContext to conditionally render MyStep component based on data source type, improving the import process for non-image datasets.

* Refactor image dataset handling by improving internationalization strings, enhancing error messages, and streamlining the preview URL generation process.

* 图片上传到新建的 dataset_collection_images 表，逻辑跟随更改

* 修改了除了controller的其他部分问题

* 把图片数据集的逻辑整合到controller里面

* 补充i18n

* 补充i18n

* resolve评论：主要是上传逻辑的更改和组件复用

* 图片名称的图标显示

* 修改编译报错的命名问题

* 删除不需要的collectionid部分

* 多余文件的处理和改动一个删除按钮

* 除了loading和统一的imageId,其他都resolve掉的

* 处理图标报错

* 复用了MyPhotoView并采用全部替换的方式将imageFileId变成imageId

* 去除不必要文件修改

* 报错和字段修改

* 增加上传成功后删除临时文件的逻辑以及回退一些修改

* 删除path字段，将图片保存到gridfs内，并修改增删等操作的代码

* 修正编译错误

---------

Co-authored-by: archer <545436317@qq.com>

* perf: image dataset

* feat: insert image

* perf: image icon

* fix: training state

---------

Co-authored-by: Zhuangzai fa <143257420+ctrlz526@users.noreply.github.com>
This commit is contained in:
Archer
2025-06-03 16:30:59 +08:00
committed by archer
parent 9fb5d05865
commit 92c38d9d2f
104 changed files with 2341 additions and 693 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
projects/app/src/pageComponents/dataset/detail/CollectionCard/BackupImportModal.tsx
View File

@@ -50,7 +50,7 @@ const BackupImportModal = ({
maxCount={1}
fileType="csv"
selectFiles={selectFiles}
setSelectFiles={setSelectFiles}
setSelectFiles={(e) => setSelectFiles(e)}
/>
{/* File render */}
{selectFiles.length > 0 && (

25
projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx
View File

@@ -248,6 +248,26 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
});
}
},
...(feConfigs?.isPlus
? [
{
label: (
<Flex>
<MyIcon name={'image'} mr={2} w={'20px'} />
{t('dataset:core.dataset.Image collection')}
</Flex>
),
onClick: () =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.import,
source: ImportDataSourceEnum.imageDataset
}
})
}
]
: []),
{
label: (
<Flex>
@@ -473,7 +493,10 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
name={editFolderData.name}
/>
)}
<EditCreateVirtualFileModal iconSrc={'modal/manualDataset'} closeBtnText={''} />
<EditCreateVirtualFileModal
iconSrc={'modal/manualDataset'}
closeBtnText={t('common:Cancel')}
/>
{isOpenFileSourceSelector && <FileSourceSelector onClose={onCloseFileSourceSelector} />}
{isOpenBackupImportModal && (
<BackupImportModal

2
projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx
View File

@@ -421,7 +421,7 @@ const AddTagToCollections = ({
() =>
collectionsList.map((item) => {
const collection = item.data;
const icon = getCollectionIcon(collection.type, collection.name);
const icon = getCollectionIcon({ type: collection.type, name: collection.name });
return {
id: collection._id,
tags: collection.tags,

107
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingStates.tsx
View File

@@ -35,6 +35,8 @@ import { useForm } from 'react-hook-form';
import type { getTrainingDetailResponse } from '@/pages/api/core/dataset/collection/trainingDetail';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import MyImage from '@/components/MyImage';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
enum TrainingStatus {
NotStart = 'NotStart',
@@ -48,6 +50,8 @@ const ProgressView = ({ trainingDetail }: { trainingDetail: getTrainingDetailRes
const { t } = useTranslation();
const isQA = trainingDetail?.trainingType === DatasetCollectionDataProcessModeEnum.qa;
const isImageParse =
trainingDetail?.trainingType === DatasetCollectionDataProcessModeEnum.imageParse;
/*
状态计算
@@ -102,6 +106,18 @@ const ProgressView = ({ trainingDetail }: { trainingDetail: getTrainingDetailRes
status: TrainingStatus.Ready,
errorCount: 0
},
...(isImageParse
? [
{
errorCount: trainingDetail.errorCounts.imageParse,
label: t(TrainingProcess.parseImage.label),
statusText: getStatusText(TrainingModeEnum.imageParse),
status: getTrainingStatus({
errorCount: trainingDetail.errorCounts.imageParse
})
}
]
: []),
...(isQA
? [
{
@@ -114,7 +130,7 @@ const ProgressView = ({ trainingDetail }: { trainingDetail: getTrainingDetailRes
}
]
: []),
...(trainingDetail?.advancedTraining.imageIndex && !isQA
...(trainingDetail?.advancedTraining.imageIndex
? [
{
errorCount: trainingDetail.errorCounts.image,
@@ -126,7 +142,7 @@ const ProgressView = ({ trainingDetail }: { trainingDetail: getTrainingDetailRes
}
]
: []),
...(trainingDetail?.advancedTraining.autoIndexes && !isQA
...(trainingDetail?.advancedTraining.autoIndexes
? [
{
errorCount: trainingDetail.errorCounts.auto,
@@ -159,7 +175,17 @@ const ProgressView = ({ trainingDetail }: { trainingDetail: getTrainingDetailRes
];
return states;
}, [trainingDetail, t, isQA]);
}, [
trainingDetail.queuedCounts,
trainingDetail.trainingCounts,
trainingDetail.errorCounts,
trainingDetail?.advancedTraining.imageIndex,
trainingDetail?.advancedTraining.autoIndexes,
trainingDetail.trainedCount,
t,
isImageParse,
isQA
]);
return (
<Flex flexDirection={'column'} gap={6}>
@@ -254,11 +280,20 @@ const ProgressView = ({ trainingDetail }: { trainingDetail: getTrainingDetailRes
);
};
const ErrorView = ({ datasetId, collectionId }: { datasetId: string; collectionId: string }) => {
const ErrorView = ({
datasetId,
collectionId,
refreshTrainingDetail
}: {
datasetId: string;
collectionId: string;
refreshTrainingDetail: () => void;
}) => {
const { t } = useTranslation();
const TrainingText = {
[TrainingModeEnum.chunk]: t('dataset:process.Vectorizing'),
[TrainingModeEnum.qa]: t('dataset:process.Get QA'),
[TrainingModeEnum.imageParse]: t('dataset:process.Image_Index'),
[TrainingModeEnum.image]: t('dataset:process.Image_Index'),
[TrainingModeEnum.auto]: t('dataset:process.Auto_Index')
};
@@ -308,6 +343,7 @@ const ErrorView = ({ datasetId, collectionId }: { datasetId: string; collectionI
manual: true,
onSuccess: () => {
refreshList();
refreshTrainingDetail();
setEditChunk(undefined);
}
}
@@ -316,6 +352,7 @@ const ErrorView = ({ datasetId, collectionId }: { datasetId: string; collectionI
if (editChunk) {
return (
<EditView
loading={updateLoading}
editChunk={editChunk}
onCancel={() => setEditChunk(undefined)}
onSave={(data) => {
@@ -401,10 +438,12 @@ const ErrorView = ({ datasetId, collectionId }: { datasetId: string; collectionI
};
const EditView = ({
loading,
editChunk,
onCancel,
onSave
}: {
loading: boolean;
editChunk: getTrainingDataDetailResponse;
onCancel: () => void;
onSave: (data: { q: string; a?: string }) => void;
@@ -419,20 +458,41 @@ const EditView = ({
return (
<Flex flexDirection={'column'} gap={4}>
{editChunk?.a && <Box>q</Box>}
<MyTextarea {...register('q')} minH={editChunk?.a ? 200 : 400} />
{editChunk?.imagePreviewUrl && (
<Box>
<FormLabel>{t('file:image')}</FormLabel>
<Box w={'100%'} h={'200px'} border={'base'} borderRadius={'md'}>
<MyImage src={editChunk.imagePreviewUrl} alt="image" w={'100%'} h={'100%'} />
</Box>
</Box>
)}
<Box>
{(editChunk?.a || editChunk?.imagePreviewUrl) && (
<FormLabel>
{editChunk?.a
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
)}
<MyTextarea
{...register('q', { required: true })}
minH={editChunk?.a || editChunk?.imagePreviewUrl ? 200 : 400}
/>
</Box>
{editChunk?.a && (
<>
<Box>a</Box>
<Box>
<Box>{t('common:dataset_data_input_a')}</Box>
<MyTextarea {...register('a')} minH={200} />
</>
</Box>
)}
<Flex justifyContent={'flex-end'} gap={4}>
<Button variant={'outline'} onClick={onCancel}>
{t('common:Cancel')}
</Button>
<Button variant={'primary'} onClick={handleSubmit(onSave)}>
{t('dataset:dataset.ReTrain')}
<Button isLoading={loading} variant={'primary'} onClick={handleSubmit(onSave)}>
{t('common:Confirm')}
</Button>
</Flex>
</Flex>
@@ -453,14 +513,15 @@ const TrainingStates = ({
const { t } = useTranslation();
const [tab, setTab] = useState<typeof defaultTab>(defaultTab);
const { data: trainingDetail, loading } = useRequest2(
() => getDatasetCollectionTrainingDetail(collectionId),
{
pollingInterval: 5000,
pollingWhenHidden: false,
manual: false
}
);
const {
data: trainingDetail,
loading,
runAsync: refreshTrainingDetail
} = useRequest2(() => getDatasetCollectionTrainingDetail(collectionId), {
pollingInterval: 5000,
pollingWhenHidden: false,
manual: false
});
const errorCounts = (Object.values(trainingDetail?.errorCounts || {}) as number[]).reduce(
(acc, count) => acc + count,
@@ -493,7 +554,13 @@ const TrainingStates = ({
]}
/>
{tab === 'states' && trainingDetail && <ProgressView trainingDetail={trainingDetail} />}
{tab === 'errors' && <ErrorView datasetId={datasetId} collectionId={collectionId} />}
{tab === 'errors' && (
<ErrorView
datasetId={datasetId}
collectionId={collectionId}
refreshTrainingDetail={refreshTrainingDetail}
/>
)}
</ModalBody>
</MyModal>
);

2
projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx
View File

@@ -75,7 +75,7 @@ const CollectionCard = () => {
const formatCollections = useMemo(
() =>
collections.map((collection) => {
const icon = getCollectionIcon(collection.type, collection.name);
const icon = getCollectionIcon({ type: collection.type, name: collection.name });
const status = (() => {
if (collection.hasError) {
return {

144
projects/app/src/pageComponents/dataset/detail/DataCard.tsx
View File

@@ -1,5 +1,5 @@
import React, { useState, useMemo } from 'react';
import { Box, Card, IconButton, Flex, Button, useTheme } from '@chakra-ui/react';
import { Box, Card, IconButton, Flex, Button, useTheme, Image } from '@chakra-ui/react';
import {
getDatasetDataList,
delOneDatasetDataById,
@@ -24,28 +24,36 @@ import TagsPopOver from './CollectionCard/TagsPopOver';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyDivider from '@fastgpt/web/components/common/MyDivider';
import Markdown from '@/components/Markdown';
import { useMemoizedFn } from 'ahooks';
import { useBoolean, useMemoizedFn } from 'ahooks';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import { TabEnum } from './NavBar';
import {
DatasetCollectionDataProcessModeEnum,
DatasetCollectionTypeEnum,
ImportDataSourceEnum
} from '@fastgpt/global/core/dataset/constants';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import TrainingStates from './CollectionCard/TrainingStates';
import { getTextValidLength } from '@fastgpt/global/common/string/utils';
import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import MyImage from '@fastgpt/web/components/common/Image/MyImage';
import dynamic from 'next/dynamic';
const InsertImagesModal = dynamic(() => import('./data/InsertImageModal'), {
ssr: false
});
const DataCard = () => {
const theme = useTheme();
const router = useRouter();
const { isPc } = useSystem();
const { collectionId = '', datasetId } = router.query as {
const { feConfigs } = useSystemStore();
const { collectionId = '' } = router.query as {
collectionId: string;
datasetId: string;
};
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { feConfigs } = useSystemStore();
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
@@ -78,21 +86,30 @@ const DataCard = () => {
const [editDataId, setEditDataId] = useState<string>();
// get file info
const { data: collection } = useRequest2(() => getDatasetCollectionById(collectionId), {
refreshDeps: [collectionId],
manual: false,
onError: () => {
router.replace({
query: {
datasetId
}
});
// Get collection info
const { data: collection, runAsync: reloadCollection } = useRequest2(
() => getDatasetCollectionById(collectionId),
{
refreshDeps: [collectionId],
manual: false,
onError: () => {
router.replace({
query: {
datasetId
}
});
}
}
});
);
const canWrite = useMemo(() => datasetDetail.permission.hasWritePer, [datasetDetail]);
const [
isInsertImagesModalOpen,
{ setTrue: openInsertImagesModal, setFalse: closeInsertImagesModal }
] = useBoolean();
const isImageCollection = collection?.type === DatasetCollectionTypeEnum.images;
const onDeleteOneData = useMemoizedFn(async (dataId: string) => {
try {
await delOneDatasetDataById(dataId);
@@ -125,6 +142,7 @@ const DataCard = () => {
>
{collection?._id && (
<RawSourceBox
collectionType={collection.type}
collectionId={collection._id}
{...getCollectionSourceData(collection)}
fontSize={['sm', 'md']}
@@ -158,7 +176,7 @@ const DataCard = () => {
{t('dataset:retain_collection')}
</Button>
)}
{canWrite && (
{canWrite && !isImageCollection && (
<Button
ml={2}
variant={'whitePrimary'}
@@ -171,6 +189,17 @@ const DataCard = () => {
{t('common:dataset.Insert Data')}
</Button>
)}
{canWrite && isImageCollection && (
<Button
ml={2}
variant={'whitePrimary'}
size={['sm', 'md']}
isDisabled={!collection}
onClick={openInsertImagesModal}
>
{t('dataset:insert_images')}
</Button>
)}
</Flex>
<Box justifyContent={'center'} px={6} pos={'relative'} w={'100%'}>
<MyDivider my={'17px'} w={'100%'} />
@@ -236,7 +265,7 @@ const DataCard = () => {
userSelect={'none'}
boxShadow={'none'}
bg={index % 2 === 1 ? 'myGray.50' : 'blue.50'}
border={theme.borders.sm}
border={'sm'}
position={'relative'}
overflow={'hidden'}
_hover={{
@@ -282,17 +311,35 @@ const DataCard = () => {
</Flex>
{/* Data content */}
<Box wordBreak={'break-all'} fontSize={'sm'}>
<Markdown source={item.q} isDisabled />
{!!item.a && (
<>
<MyDivider />
<Markdown source={item.a} isDisabled />
</>
)}
</Box>
{item.imagePreviewUrl ? (
<Box display={['block', 'flex']} alignItems={'center'} gap={[3, 6]}>
<Box flex="1 0 0">
<MyImage
src={item.imagePreviewUrl}
alt={''}
w={'100%'}
h="100%"
maxH={'300px'}
objectFit="contain"
/>
</Box>
<Box flex="1 0 0" maxH={'300px'} overflow={'hidden'} fontSize="sm">
<Markdown source={item.q} isDisabled />
</Box>
</Box>
) : (
<Box wordBreak={'break-all'} fontSize={'sm'}>
<Markdown source={item.q} isDisabled />
{!!item.a && (
<>
<MyDivider />
<Markdown source={item.a} isDisabled />
</>
)}
</Box>
)}
{/* Mask */}
{/* Footer */}
<Flex
className="footer"
position={'absolute'}
@@ -317,17 +364,23 @@ const DataCard = () => {
py={1}
mr={2}
>
<MyIcon
bg={'white'}
color={'myGray.600'}
borderRadius={'sm'}
border={'1px'}
borderColor={'myGray.200'}
name="common/text/t"
w={'14px'}
mr={1}
/>
{getTextValidLength(item.q + item.a || '')}
{item.imageSize ? (
<>{formatFileSize(item.imageSize)}</>
) : (
<>
<MyIcon
bg={'white'}
color={'myGray.600'}
borderRadius={'sm'}
border={'1px'}
borderColor={'myGray.200'}
name="common/text/t"
w={'14px'}
mr={1}
/>
{getTextValidLength((item?.q || '') + (item?.a || ''))}
</>
)}
</Flex>
{canWrite && (
<PopoverConfirm
@@ -362,7 +415,7 @@ const DataCard = () => {
collectionId={collection._id}
dataId={editDataId}
onClose={() => setEditDataId(undefined)}
onSuccess={(data) => {
onSuccess={(data: any) => {
if (editDataId === '') {
refreshList();
return;
@@ -386,9 +439,16 @@ const DataCard = () => {
datasetId={datasetId}
defaultTab={'errors'}
collectionId={errorModalId}
onClose={() => setErrorModalId('')}
onClose={() => {
setErrorModalId('');
refreshList();
reloadCollection();
}}
/>
)}
{isInsertImagesModalOpen && (
<InsertImagesModal collectionId={collectionId} onClose={closeInsertImagesModal} />
)}
</MyBox>
);
};

42
projects/app/src/pageComponents/dataset/detail/Import/Context.tsx
View File

@@ -173,6 +173,20 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
{
title: t('dataset:import_confirm')
}
],
[ImportDataSourceEnum.imageDataset]: [
{
title: t('dataset:import_select_file')
},
{
title: t('dataset:import_param_setting')
},
{
title: t('dataset:import_data_preview')
},
{
title: t('dataset:import_confirm')
}
]
};
const steps = modeSteps[source];
@@ -238,20 +252,22 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
<Box flex={1} />
</Flex>
{/* step */}
<Box
mt={4}
mb={5}
px={3}
py={[2, 4]}
bg={'myGray.50'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
borderRadius={'md'}
>
<Box maxW={['100%', '900px']} mx={'auto'}>
<MyStep />
{source !== ImportDataSourceEnum.imageDataset && (
<Box
mt={4}
mb={5}
px={3}
py={[2, 4]}
bg={'myGray.50'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
borderRadius={'md'}
>
<Box maxW={['100%', '900px']} mx={'auto'}>
<MyStep />
</Box>
</Box>
</Box>
)}
{children}
</DatasetImportContext.Provider>
);

107
projects/app/src/pageComponents/dataset/detail/Import/components/FileSelector.tsx
View File

@@ -7,15 +7,8 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import React, { type DragEvent, useCallback, useMemo, useState } from 'react';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { uploadFile2DB } from '@/web/common/file/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import type { ImportSourceItemType } from '@/web/core/dataset/type';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { getErrText } from '@fastgpt/global/common/error/utils';
export type SelectFileItemType = {
fileId: string;
@@ -26,23 +19,18 @@ export type SelectFileItemType = {
const FileSelector = ({
fileType,
selectFiles,
setSelectFiles,
onStartSelect,
onFinishSelect,
onSelectFiles,
...props
}: {
fileType: string;
selectFiles: ImportSourceItemType[];
setSelectFiles: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
onStartSelect: () => void;
onFinishSelect: () => void;
onSelectFiles: (e: SelectFileItemType[]) => any;
} & FlexProps) => {
const { t } = useTranslation();
const { toast } = useToast();
const { feConfigs } = useSystemStore();
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
const maxCount = feConfigs?.uploadFileMaxAmount || 1000;
const maxSize = (feConfigs?.uploadFileMaxSize || 1024) * 1024 * 1024;
@@ -65,90 +53,6 @@ const FileSelector = ({
'i'
);
const { runAsync: onSelectFile, loading: isLoading } = useRequest2(
async (files: SelectFileItemType[]) => {
{
await Promise.all(
files.map(async ({ fileId, file }) => {
try {
const { fileId: uploadFileId } = await uploadFile2DB({
file,
bucketName: BucketNameEnum.dataset,
data: {
datasetId
},
percentListen: (e) => {
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
uploadedFileRate: item.uploadedFileRate
? Math.max(e, item.uploadedFileRate)
: e
}
: item
)
);
}
});
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
dbFileId: uploadFileId,
isUploading: false,
uploadedFileRate: 100
}
: item
)
);
} catch (error) {
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
isUploading: false,
errorMsg: getErrText(error)
}
: item
)
);
}
})
);
}
},
{
onBefore([files]) {
onStartSelect();
setSelectFiles((state) => {
const formatFiles = files.map<ImportSourceItemType>((selectFile) => {
const { fileId, file } = selectFile;
return {
id: fileId,
createStatus: 'waiting',
file,
sourceName: file.name,
sourceSize: formatFileSize(file.size),
icon: getFileIcon(file.name),
isUploading: true,
uploadedFileRate: 0
};
});
const results = formatFiles.concat(state).slice(0, maxCount);
return results;
});
},
onFinally() {
onFinishSelect();
}
}
);
const selectFileCallback = useCallback(
(files: SelectFileItemType[]) => {
if (selectFiles.length + files.length > maxCount) {
@@ -160,7 +64,7 @@ const FileSelector = ({
}
// size check
if (!maxSize) {
return onSelectFile(files);
return onSelectFiles(files);
}
const filterFiles = files.filter((item) => item.file.size <= maxSize);
@@ -171,9 +75,9 @@ const FileSelector = ({
});
}
return onSelectFile(filterFiles);
return onSelectFiles(filterFiles);
},
[t, maxCount, maxSize, onSelectFile, selectFiles.length, toast]
[t, maxCount, maxSize, onSelectFiles, selectFiles.length, toast]
);
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
@@ -278,7 +182,6 @@ const FileSelector = ({
return (
<MyBox
isLoading={isLoading}
display={'flex'}
flexDirection={'column'}
alignItems={'center'}

4
projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileCustomText.tsx
View File

@@ -71,7 +71,7 @@ const CustomTextInput = () => {
<Box maxW={['100%', '800px']}>
<Box display={['block', 'flex']} alignItems={'center'}>
<Box flex={'0 0 120px'} fontSize={'sm'}>
{t('common:core.dataset.collection.Collection name')}
{t('dataset:collection_name')}
</Box>
<Input
flex={'1 0 0'}
@@ -79,7 +79,7 @@ const CustomTextInput = () => {
{...register('name', {
required: true
})}
placeholder={t('common:core.dataset.collection.Collection name')}
placeholder={t('dataset:collection_name')}
bg={'myGray.50'}
/>
</Box>

103
projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLocal.tsx
View File

@@ -1,14 +1,20 @@
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { type ImportSourceItemType } from '@/web/core/dataset/type.d';
import { Box, Button } from '@chakra-ui/react';
import FileSelector from '../components/FileSelector';
import FileSelector, { type SelectFileItemType } from '../components/FileSelector';
import { useTranslation } from 'next-i18next';
import dynamic from 'next/dynamic';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { RenderUploadFiles } from '../components/RenderFiles';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { uploadFile2DB } from '@/web/common/file/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'));
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
@@ -33,14 +39,16 @@ export default React.memo(FileLocal);
const SelectFile = React.memo(function SelectFile() {
const { t } = useTranslation();
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>(
sources.map((source) => ({
isUploading: false,
...source
}))
);
const [uploading, setUploading] = useState(false);
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
useEffect(() => {
@@ -53,15 +61,90 @@ const SelectFile = React.memo(function SelectFile() {
goToNext();
}, [goToNext]);
const { runAsync: onSelectFiles, loading: uploading } = useRequest2(
async (files: SelectFileItemType[]) => {
{
await Promise.all(
files.map(async ({ fileId, file }) => {
try {
const { fileId: uploadFileId } = await uploadFile2DB({
file,
bucketName: BucketNameEnum.dataset,
data: {
datasetId
},
percentListen: (e) => {
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
uploadedFileRate: item.uploadedFileRate
? Math.max(e, item.uploadedFileRate)
: e
}
: item
)
);
}
});
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
dbFileId: uploadFileId,
isUploading: false,
uploadedFileRate: 100
}
: item
)
);
} catch (error) {
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
isUploading: false,
errorMsg: getErrText(error)
}
: item
)
);
}
})
);
}
},
{
onBefore([files]) {
setSelectFiles((state) => {
return [
...state,
...files.map<ImportSourceItemType>((selectFile) => {
const { fileId, file } = selectFile;
return {
id: fileId,
createStatus: 'waiting',
file,
sourceName: file.name,
sourceSize: formatFileSize(file.size),
icon: getFileIcon(file.name),
isUploading: true,
uploadedFileRate: 0
};
})
];
});
}
}
);
return (
<Box>
<FileSelector
fileType={fileType}
selectFiles={selectFiles}
setSelectFiles={setSelectFiles}
onStartSelect={() => setUploading(true)}
onFinishSelect={() => setUploading(false)}
/>
<FileSelector fileType={fileType} selectFiles={selectFiles} onSelectFiles={onSelectFiles} />
{/* render files */}
<RenderUploadFiles files={selectFiles} setFiles={setSelectFiles} />

189
projects/app/src/pageComponents/dataset/detail/Import/diffSource/ImageDataset.tsx Normal file
View File

@@ -0,0 +1,189 @@
import React, { useState } from 'react';
import { Box, Button, Flex, Input, Image } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { useRouter } from 'next/router';
import { TabEnum } from '../../NavBar';
import { createImageDatasetCollection } from '@/web/core/dataset/image/api';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import { useForm } from 'react-hook-form';
import FileSelector, { type SelectFileItemType } from '../components/FileSelector';
import type { ImportSourceItemType } from '@/web/core/dataset/type';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { DatasetImportContext } from '../Context';
import MyImage from '@fastgpt/web/components/common/Image/MyImage';
const fileType = '.jpg, .jpeg, .png';
const ImageDataset = () => {
return <SelectFile />;
};
export default React.memo(ImageDataset);
const SelectFile = React.memo(function SelectFile() {
const { t } = useTranslation();
const router = useRouter();
const parentId = useContextSelector(DatasetImportContext, (v) => v.parentId);
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>([]);
const [uploadProgress, setUploadProgress] = useState(0);
const { register, handleSubmit } = useForm({
defaultValues: {
name: ''
}
});
const onSelectFiles = (files: SelectFileItemType[]) => {
setSelectFiles((pre) => {
const formatFiles = Array.from(files).map<ImportSourceItemType>((item) => {
const previewUrl = URL.createObjectURL(item.file);
return {
id: getNanoid(),
createStatus: 'waiting',
file: item.file,
sourceName: item.file.name,
icon: previewUrl
};
});
return [...pre, ...formatFiles];
});
};
const onRemoveFile = (index: number) => {
setSelectFiles((prev) => {
return prev.filter((_, i) => i !== index);
});
};
const { runAsync: onCreate, loading: creating } = useRequest2(
async ({ name: collectionName }: { name: string }) => {
return await createImageDatasetCollection({
parentId,
datasetId,
collectionName,
files: selectFiles.map((item) => item.file!).filter(Boolean),
onUploadProgress: setUploadProgress
});
},
{
manual: true,
successToast: t('common:create_success'),
onSuccess() {
router.replace({
query: {
datasetId: router.query.datasetId,
currentTab: TabEnum.collectionCard
}
});
}
}
);
return (
<Flex flexDirection={'column'} maxW={'850px'} mx={'auto'} mt={7}>
<Flex alignItems="center" width="100%">
<FormLabel required width={['100px', '140px']}>
{t('dataset:collection_name')}
</FormLabel>
<Input
flex="0 0 400px"
bg="myGray.50"
placeholder={t('dataset:collection_name')}
{...register('name', { required: true })}
/>
</Flex>
<Flex mt={7} alignItems="flex-start" width="100%">
<FormLabel required width={['100px', '140px']}>
{t('common:core.dataset.collection.Collection raw text')}
</FormLabel>
<Box flex={'1 0 0'}>
<Box>
<FileSelector
fileType={fileType}
selectFiles={selectFiles}
onSelectFiles={onSelectFiles}
/>
</Box>
{selectFiles.length > 0 && (
<Flex flexWrap={'wrap'} gap={4} mt={3} width="100%">
{selectFiles.map((file, index) => (
<Box
key={index}
w="100px"
h={'100px'}
position={'relative'}
_hover={{
'.close-icon': { display: 'block' }
}}
bg={'myGray.50'}
borderRadius={'md'}
border={'base'}
borderStyle={'dashed'}
p={1}
>
<MyImage
src={file.icon}
w="100%"
h={'100%'}
objectFit={'contain'}
alt={file.sourceName}
/>
<MyIcon
name={'closeSolid'}
w={'1rem'}
h={'1rem'}
color={'myGray.700'}
cursor={'pointer'}
_hover={{ color: 'red.500' }}
position={'absolute'}
rounded={'full'}
bg={'white'}
right={'-8px'}
top={'-2px'}
onClick={() => onRemoveFile(index)}
className="close-icon"
display={['', 'none']}
zIndex={10}
/>
</Box>
))}
</Flex>
)}
</Box>
</Flex>
<Flex width="100%" justifyContent="flex-end" mt="9">
<Button isDisabled={selectFiles.length === 0 || creating} onClick={handleSubmit(onCreate)}>
{creating ? (
uploadProgress >= 100 ? (
<Box>{t('dataset:images_creating')}</Box>
) : (
<Box>{t('dataset:uploading_progress', { num: uploadProgress })}</Box>
)
) : selectFiles.length > 0 ? (
<>
<Box>
{t('dataset:confirm_import_images', {
num: selectFiles.length
})}
</Box>
</>
) : (
<Box>{t('common:comfirn_create')}</Box>
)}
</Button>
</Flex>
</Flex>
);
});

2
projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx
View File

@@ -37,7 +37,7 @@ const ReTraining = () => {
apiFileId: collection.apiFileId,
createStatus: 'waiting',
icon: getCollectionIcon(collection.type, collection.name),
icon: getCollectionIcon({ type: collection.type, name: collection.name }),
id: collection._id,
isUploading: false,
sourceName: collection.name,

3
projects/app/src/pageComponents/dataset/detail/Import/index.tsx
View File

@@ -11,6 +11,7 @@ const FileCustomText = dynamic(() => import('./diffSource/FileCustomText'));
const ExternalFileCollection = dynamic(() => import('./diffSource/ExternalFile'));
const APIDatasetCollection = dynamic(() => import('./diffSource/APIDataset'));
const ReTraining = dynamic(() => import('./diffSource/ReTraining'));
const ImageDataset = dynamic(() => import('./diffSource/ImageDataset'));
const ImportDataset = () => {
const importSource = useContextSelector(DatasetImportContext, (v) => v.importSource);
@@ -22,6 +23,8 @@ const ImportDataset = () => {
if (importSource === ImportDataSourceEnum.fileCustom) return FileCustomText;
if (importSource === ImportDataSourceEnum.externalFile) return ExternalFileCollection;
if (importSource === ImportDataSourceEnum.apiDataset) return APIDatasetCollection;
if (importSource === ImportDataSourceEnum.imageDataset) return ImageDataset;
return null;
}, [importSource]);
return ImportComponent ? (

226
projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx
View File

@@ -1,37 +1,39 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Box, Flex, Button, Textarea, ModalFooter, HStack, VStack } from '@chakra-ui/react';
import { type UseFormRegister, useFieldArray, useForm } from 'react-hook-form';
import { Box, Flex, Button, Textarea, ModalFooter, HStack, VStack, Image } from '@chakra-ui/react';
import type { UseFormRegister } from 'react-hook-form';
import { useFieldArray, useForm } from 'react-hook-form';
import {
postInsertData2Dataset,
putDatasetDataById,
getDatasetCollectionById,
getDatasetDataItemById
} from '@/web/core/dataset/api';
import { useToast } from '@fastgpt/web/hooks/useToast';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useTranslation } from 'next-i18next';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
import { type DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import type { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
import DeleteIcon from '@fastgpt/web/components/common/Icon/delete';
import { defaultCollectionDetail } from '@/web/core/dataset/constants';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import styles from './styles.module.scss';
import {
DatasetDataIndexTypeEnum,
getDatasetIndexMapData
} from '@fastgpt/global/core/dataset/data/constants';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constants';
import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyIconButton from '@fastgpt/web/components/common/Icon/button';
import MyImage from '@/components/MyImage/index';
export type InputDataType = {
q: string;
a: string;
imagePreivewUrl?: string;
indexes: (Omit<DatasetDataIndexItemType, 'dataId'> & {
dataId?: string; // pg data id
fold: boolean;
@@ -40,7 +42,8 @@ export type InputDataType = {
enum TabEnum {
chunk = 'chunk',
qa = 'qa'
qa = 'qa',
image = 'image'
}
const InputDataModal = ({
@@ -52,17 +55,16 @@ const InputDataModal = ({
}: {
collectionId: string;
dataId?: string;
defaultValue?: { q: string; a?: string };
defaultValue?: { q?: string; a?: string; imagePreivewUrl?: string };
onClose: () => void;
onSuccess: (data: InputDataType & { dataId: string }) => void;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { embeddingModelList, defaultModels } = useSystemStore();
const [currentTab, setCurrentTab] = useState(TabEnum.chunk);
const [currentTab, setCurrentTab] = useState<TabEnum>();
const { register, handleSubmit, reset, control } = useForm<InputDataType>();
const { register, handleSubmit, reset, control, watch } = useForm<InputDataType>();
const {
fields: indexes,
prepend: prependIndexes,
@@ -72,16 +74,24 @@ const InputDataModal = ({
control,
name: 'indexes'
});
const imagePreivewUrl = watch('imagePreivewUrl');
const { data: collection = defaultCollectionDetail } = useRequest2(
() => {
return getDatasetCollectionById(collectionId);
},
() => getDatasetCollectionById(collectionId),
{
manual: false,
refreshDeps: [collectionId]
refreshDeps: [collectionId],
onSuccess(res) {
if (res.type === DatasetCollectionTypeEnum.images) {
setCurrentTab(TabEnum.image);
} else {
setCurrentTab(TabEnum.chunk);
}
}
}
);
// Get data
const { loading: isFetchingData } = useRequest2(
async () => {
if (dataId) return getDatasetDataItemById(dataId);
@@ -93,8 +103,9 @@ const InputDataModal = ({
onSuccess(res) {
if (res) {
reset({
q: res.q,
a: res.a,
q: res.q || '',
a: res.a || '',
imagePreivewUrl: res.imagePreivewUrl,
indexes: res.indexes.map((item) => ({
...item,
fold: true
@@ -102,54 +113,32 @@ const InputDataModal = ({
});
} else if (defaultValue) {
reset({
q: defaultValue.q,
a: defaultValue.a
q: defaultValue.q || '',
a: defaultValue.a || '',
imagePreivewUrl: defaultValue.imagePreivewUrl
});
}
if (res?.a || defaultValue?.a) {
setCurrentTab(TabEnum.qa);
}
},
onError(err) {
toast({
status: 'error',
title: t(getErrText(err) as any)
});
onClose();
}
}
);
const maxToken = useMemo(() => {
const vectorModel =
embeddingModelList.find((item) => item.model === collection.dataset.vectorModel) ||
defaultModels.embedding;
return vectorModel?.maxToken || 3000;
}, [collection.dataset.vectorModel, defaultModels.embedding, embeddingModelList]);
// import new data
// Import new data
const { runAsync: sureImportData, loading: isImporting } = useRequest2(
async (e: InputDataType) => {
if (!e.q) {
return Promise.reject(t('common:dataset.data.input is empty'));
}
const totalLength = e.q.length + (e.a?.length || 0);
if (totalLength >= maxToken * 1.4) {
return Promise.reject(t('common:core.dataset.data.Too Long'));
}
const data = { ...e };
const dataId = await postInsertData2Dataset({
const postData: any = {
collectionId: collection._id,
q: e.q,
a: currentTab === TabEnum.qa ? e.a : '',
// Contains no default index
indexes: e.indexes?.filter((item) => !!item.text?.trim())
});
indexes: e.indexes.filter((item) => !!item.text?.trim())
};
const dataId = await postInsertData2Dataset(postData);
return {
...data,
@@ -166,23 +155,26 @@ const InputDataModal = ({
a: '',
indexes: []
});
onSuccess(e);
},
errorToast: t('common:error.unKnow')
errorToast: t('dataset:common.error.unKnow')
}
);
// update
// Update data
const { runAsync: onUpdateData, loading: isUpdating } = useRequest2(
async (e: InputDataType) => {
if (!dataId) return Promise.reject(t('common:error.unKnow'));
await putDatasetDataById({
const updateData: any = {
dataId,
q: e.q,
a: currentTab === TabEnum.qa ? e.a : '',
indexes: e.indexes.filter((item) => !!item.text?.trim())
});
};
await putDatasetDataById(updateData);
return {
dataId,
@@ -202,10 +194,18 @@ const InputDataModal = ({
const isLoading = isFetchingData;
const icon = useMemo(
() => getSourceNameIcon({ sourceName: collection.sourceName, sourceId: collection.sourceId }),
() => getCollectionIcon({ type: collection.type, name: collection.sourceName }),
[collection]
);
const maxToken = useMemo(() => {
const vectorModel =
embeddingModelList.find((item) => item.model === collection.dataset.vectorModel) ||
defaultModels.embedding;
return vectorModel?.maxToken || 2000;
}, [collection.dataset.vectorModel, defaultModels.embedding, embeddingModelList]);
return (
<MyModal
isOpen={true}
@@ -243,17 +243,19 @@ const InputDataModal = ({
>
{/* Tab */}
<Box px={[5, '3.25rem']}>
<FillRowTabs
list={[
{ label: t('common:dataset_data_input_chunk'), value: TabEnum.chunk },
{ label: t('common:dataset_data_input_qa'), value: TabEnum.qa }
]}
py={1}
value={currentTab}
onChange={(e) => {
setCurrentTab(e);
}}
/>
{(currentTab === TabEnum.chunk || currentTab === TabEnum.qa) && (
<FillRowTabs
list={[
{ label: t('common:dataset_data_input_chunk'), value: TabEnum.chunk },
{ label: t('common:dataset_data_input_qa'), value: TabEnum.qa }
]}
py={1}
value={currentTab}
onChange={(e) => {
setCurrentTab(e);
}}
/>
)}
</Box>
<Flex flex={'1 0 0'} h={['auto', '0']} gap={6} flexDir={['column', 'row']} px={[5, '0']}>
@@ -268,45 +270,64 @@ const InputDataModal = ({
w={['100%', 0]}
overflow={['unset', 'auto']}
>
<Flex flexDir={'column'} h={'100%'}>
<FormLabel required mb={1} h={'30px'}>
{currentTab === TabEnum.chunk
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
<Textarea
resize={'none'}
placeholder={t('common:dataset_data_import_q_placeholder', { maxToken })}
className={styles.scrollbar}
maxLength={maxToken}
flex={'1 0 0'}
tabIndex={1}
_focus={{
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
bg={'myGray.25'}
borderRadius={'md'}
borderColor={'myGray.200'}
{...register(`q`, {
required: true
})}
/>
<Flex flexDir={'column'} flex={'1 0 0'} h={0}>
{currentTab === TabEnum.image && (
<>
<FormLabel required mb={1} h={'30px'}>
{t('file:image')}
</FormLabel>
<Box flex={'1 0 0'} h={0} w="100%">
<Box height="100%" position="relative" border="base" borderRadius={'md'} p={1}>
<MyImage
src={imagePreivewUrl}
h="100%"
w="100%"
objectFit="contain"
alt={t('file:Image_Preview')}
/>
</Box>
</Box>
</>
)}
{(currentTab === TabEnum.chunk || currentTab === TabEnum.qa) && (
<>
<FormLabel required mb={1} h={'30px'}>
{currentTab === TabEnum.chunk
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
<Textarea
resize={'none'}
className={styles.scrollbar}
flex={'1 0 0'}
tabIndex={1}
_focus={{
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
bg={'myGray.25'}
borderRadius={'md'}
borderColor={'myGray.200'}
{...register(`q`, {
required: true
})}
/>
</>
)}
</Flex>
{currentTab === TabEnum.qa && (
<Flex flexDir={'column'} h={'100%'}>
<Flex flexDir={'column'} flex={'1 0 0'}>
<FormLabel required mb={1}>
{t('common:dataset_data_input_a')}
</FormLabel>
<Textarea
resize={'none'}
placeholder={t('common:dataset_data_import_q_placeholder', { maxToken })}
className={styles.scrollbar}
flex={'1 0 0'}
tabIndex={1}
bg={'myGray.25'}
maxLength={maxToken}
borderRadius={'md'}
border={'1.5px solid '}
borderColor={'myGray.200'}
@@ -314,6 +335,27 @@ const InputDataModal = ({
/>
</Flex>
)}
{currentTab === TabEnum.image && (
<Flex flexDir={'column'} flex={'1 0 0'}>
<FormLabel required mb={1}>
{t('file:image_description')}
</FormLabel>
<Textarea
resize={'none'}
placeholder={t('file:image_description_tip')}
className={styles.scrollbar}
flex={'1 0 0'}
tabIndex={1}
bg={'myGray.25'}
borderRadius={'md'}
border={'1.5px solid '}
borderColor={'myGray.200'}
{...register('q', {
required: true
})}
/>
</Flex>
)}
</Flex>
{/* Index */}
<Box

78
projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx
View File

@@ -9,7 +9,8 @@ import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
import {
DatasetCollectionDataProcessModeMap,
DatasetCollectionTypeMap
DatasetCollectionTypeMap,
DatasetCollectionTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource';
import MyIcon from '@fastgpt/web/components/common/Icon';
@@ -38,6 +39,7 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
manual: false
}
);
const metadataList = useMemo<{ label?: string; value?: any }[]>(() => {
if (!collection) return [];
@@ -49,13 +51,17 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
value: t(DatasetCollectionTypeMap[collection.type]?.name as any)
},
{
label: t('common:core.dataset.collection.metadata.source name'),
label: t('dataset:collection_name'),
value: collection.file?.filename || collection?.rawLink || collection?.name
},
{
label: t('common:core.dataset.collection.metadata.source size'),
value: collection.file ? formatFileSize(collection.file.length) : '-'
},
...(collection.file
? [
{
label: t('common:core.dataset.collection.metadata.source size'),
value: formatFileSize(collection.file.length)
}
]
: []),
{
label: t('common:core.dataset.collection.metadata.Createtime'),
value: formatTime2YMDHM(collection.createTime)
@@ -64,18 +70,30 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
label: t('common:core.dataset.collection.metadata.Updatetime'),
value: formatTime2YMDHM(collection.updateTime)
},
{
label: t('dataset:collection_metadata_custom_pdf_parse'),
value: collection.customPdfParse ? 'Yes' : 'No'
},
{
label: t('common:core.dataset.collection.metadata.Raw text length'),
value: collection.rawTextLength ?? '-'
},
{
label: t('dataset:collection.training_type'),
value: t(DatasetCollectionDataProcessModeMap[collection.trainingType]?.label as any)
},
...(collection.customPdfParse !== undefined
? [
{
label: t('dataset:collection_metadata_custom_pdf_parse'),
value: collection.customPdfParse ? 'Yes' : 'No'
}
]
: []),
...(collection.rawTextLength !== undefined
? [
{
label: t('common:core.dataset.collection.metadata.Raw text length'),
value: collection.rawTextLength
}
]
: []),
...(DatasetCollectionDataProcessModeMap[collection.trainingType]
? [
{
label: t('dataset:collection.training_type'),
value: t(DatasetCollectionDataProcessModeMap[collection.trainingType]?.label as any)
}
]
: []),
...(collection.imageIndex !== undefined
? [
{
@@ -92,7 +110,7 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
}
]
: []),
...(collection.chunkSize
...(collection.chunkSize !== undefined
? [
{
label: t('dataset:chunk_size'),
@@ -100,7 +118,7 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
}
]
: []),
...(collection.indexSize
...(collection.indexSize !== undefined
? [
{
label: t('dataset:index_size'),
@@ -108,7 +126,7 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
}
]
: []),
...(webSelector
...(webSelector !== undefined
? [
{
label: t('common:core.dataset.collection.metadata.Web page selector'),
@@ -116,16 +134,14 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
}
]
: []),
{
...(collection.tags
? [
{
label: t('dataset:collection_tags'),
value: collection.tags?.join(', ') || '-'
}
]
: [])
}
...(collection.tags
? [
{
label: t('dataset:collection_tags'),
value: collection.tags?.join(', ') || '-'
}
]
: [])
];
}, [collection, t]);

2
projects/app/src/pageComponents/dataset/detail/Test.tsx
View File

@@ -456,7 +456,7 @@ const TestResults = React.memo(function TestResults({
<Box mt={1} gap={4}>
{datasetTestItem?.results.map((item, index) => (
<Box key={item.id} p={3} borderRadius={'lg'} bg={'myGray.100'} _notLast={{ mb: 2 }}>
<QuoteItem quoteItem={item} canViewSource />
<QuoteItem quoteItem={item} canViewSource canEditData />
</Box>
))}
</Box>

10
projects/app/src/pageComponents/dataset/detail/components/FileSelector.tsx
View File

@@ -25,7 +25,7 @@ const FileSelector = ({
}: {
fileType: string;
selectFiles: SelectFileItemType[];
setSelectFiles: React.Dispatch<React.SetStateAction<SelectFileItemType[]>>;
setSelectFiles: (files: SelectFileItemType[]) => void;
maxCount?: number;
} & FlexProps) => {
const { t } = useTranslation();
@@ -62,11 +62,11 @@ const FileSelector = ({
name: file.name,
size: formatFileSize(file.size)
}));
setSelectFiles((state) => {
return [...fileList, ...state].slice(0, maxCount);
});
const newFiles = [...fileList, ...selectFiles].slice(0, maxCount);
setSelectFiles(newFiles);
},
[maxCount, setSelectFiles]
[maxCount, selectFiles, setSelectFiles]
);
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {

138
projects/app/src/pageComponents/dataset/detail/data/InsertImageModal.tsx Normal file
View File

@@ -0,0 +1,138 @@
import MyModal from '@fastgpt/web/components/common/MyModal';
import React, { useState } from 'react';
import { useTranslation } from 'next-i18next';
import { Box, Button, Flex, ModalBody, ModalFooter } from '@chakra-ui/react';
import FileSelector, { type SelectFileItemType } from '../components/FileSelector';
import MyImage from '@/components/MyImage';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { insertImagesToCollection } from '@/web/core/dataset/image/api';
const fileType = '.jpg, .jpeg, .png';
type MySelectFileItemType = SelectFileItemType & { previewUrl: string };
const InsertImageModal = ({
collectionId,
onClose
}: {
collectionId: string;
onClose: () => void;
}) => {
const { t } = useTranslation();
const [selectFiles, setSelectFiles] = useState<MySelectFileItemType[]>([]);
const onSelectFiles = (files: SelectFileItemType[]) => {
setSelectFiles((pre) => {
const formatFiles = Array.from(files).map<MySelectFileItemType>((item) => {
const previewUrl = URL.createObjectURL(item.file);
return {
...item,
previewUrl
};
});
return [...pre, ...formatFiles];
});
};
const onRemoveFile = (index: number) => {
setSelectFiles((prev) => {
return prev.filter((_, i) => i !== index);
});
};
const [uploadProgress, setUploadProgress] = useState(0);
const { runAsync: onInsertImages, loading: inserting } = useRequest2(
async () => {
return await insertImagesToCollection({
collectionId,
files: selectFiles.map((item) => item.file!).filter(Boolean),
onUploadProgress: setUploadProgress
});
},
{
manual: true,
successToast: t('dataset:insert_images_success'),
onSuccess() {
onClose();
}
}
);
return (
<MyModal
isOpen
iconSrc="core/dataset/imageFill"
title={t('dataset:insert_images')}
maxW={['90vw', '605px']}
>
<ModalBody userSelect={'none'}>
<Box>
<FileSelector
fileType={fileType}
selectFiles={selectFiles}
setSelectFiles={onSelectFiles}
/>
</Box>
{selectFiles.length > 0 && (
<Flex flexWrap={'wrap'} gap={3} mt={3} width="100%">
{selectFiles.map((file, index) => (
<Box
key={index}
w="100px"
h={'100px'}
position={'relative'}
_hover={{
'.close-icon': { display: 'block' }
}}
bg={'myGray.50'}
borderRadius={'md'}
border={'base'}
borderStyle={'dashed'}
p={1}
>
<MyImage src={file.previewUrl} w="100%" h={'100%'} objectFit={'contain'} />
<MyIcon
name={'closeSolid'}
w={'1rem'}
h={'1rem'}
color={'myGray.700'}
cursor={'pointer'}
_hover={{ color: 'red.500' }}
position={'absolute'}
rounded={'full'}
bg={'white'}
right={'-8px'}
top={'-2px'}
onClick={() => onRemoveFile(index)}
className="close-icon"
display={['', 'none']}
zIndex={10}
/>
</Box>
))}
</Flex>
)}
</ModalBody>
<ModalFooter>
<Button isDisabled={inserting} variant={'whitePrimary'} mr={4} onClick={onClose}>
{t('common:Cancel')}
</Button>
<Button
isDisabled={selectFiles.length === 0 || inserting}
variant={'primary'}
onClick={onInsertImages}
>
{inserting ? (
<Box>{t('dataset:uploading_progress', { num: uploadProgress })}</Box>
) : (
<Box>{t('common:Confirm')}</Box>
)}
</Button>
</ModalFooter>
</MyModal>
);
};
export default InsertImageModal;