feat: 训练数据管理
This commit is contained in:
archer
65
src/pages/data/components/CreateDataModal.tsx
Normal file
65
src/pages/data/components/CreateDataModal.tsx
Normal file
@@ -0,0 +1,65 @@
|
||||
import React, { useState } from 'react';
|
||||
import {
|
||||
Modal,
|
||||
ModalOverlay,
|
||||
ModalContent,
|
||||
ModalHeader,
|
||||
ModalFooter,
|
||||
ModalBody,
|
||||
ModalCloseButton,
|
||||
Button,
|
||||
Input
|
||||
} from '@chakra-ui/react';
|
||||
import { postData } from '@/api/data';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
|
||||
const CreateDataModal = ({
|
||||
onClose,
|
||||
onSuccess
|
||||
}: {
|
||||
onClose: () => void;
|
||||
onSuccess: () => void;
|
||||
}) => {
|
||||
const [inputVal, setInputVal] = useState('');
|
||||
|
||||
const { isLoading, mutate } = useMutation({
|
||||
mutationFn: (name: string) => postData(name),
|
||||
onSuccess() {
|
||||
onSuccess();
|
||||
onClose();
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<Modal isOpen={true} onClose={onClose}>
|
||||
<ModalOverlay />
|
||||
<ModalContent>
|
||||
<ModalHeader>创建数据集</ModalHeader>
|
||||
<ModalCloseButton />
|
||||
|
||||
<ModalBody display={'flex'}>
|
||||
<Input
|
||||
value={inputVal}
|
||||
onChange={(e) => setInputVal(e.target.value)}
|
||||
placeholder={'数据集名称'}
|
||||
></Input>
|
||||
</ModalBody>
|
||||
<ModalFooter>
|
||||
<Button colorScheme={'gray'} onClick={onClose}>
|
||||
取消
|
||||
</Button>
|
||||
<Button
|
||||
ml={3}
|
||||
isDisabled={inputVal === ''}
|
||||
isLoading={isLoading}
|
||||
onClick={() => mutate(inputVal)}
|
||||
>
|
||||
确认
|
||||
</Button>
|
||||
</ModalFooter>
|
||||
</ModalContent>
|
||||
</Modal>
|
||||
);
|
||||
};
|
||||
|
||||
export default CreateDataModal;
|
||||
175
src/pages/data/components/ImportDataModal.tsx
Normal file
175
src/pages/data/components/ImportDataModal.tsx
Normal file
@@ -0,0 +1,175 @@
|
||||
import React, { useState, useCallback } from 'react';
|
||||
import {
|
||||
Modal,
|
||||
ModalOverlay,
|
||||
ModalContent,
|
||||
ModalHeader,
|
||||
ModalFooter,
|
||||
ModalBody,
|
||||
ModalCloseButton,
|
||||
Button,
|
||||
Input,
|
||||
Box,
|
||||
Flex,
|
||||
Textarea
|
||||
} from '@chakra-ui/react';
|
||||
import { useTabs } from '@/hooks/useTabs';
|
||||
import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { useSelectFile } from '@/hooks/useSelectFile';
|
||||
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/tools';
|
||||
import { postSplitData } from '@/api/data';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import { useLoading } from '@/hooks/useLoading';
|
||||
|
||||
const ImportDataModal = ({ dataId, onClose }: { dataId: string; onClose: () => void }) => {
|
||||
const { openConfirm, ConfirmChild } = useConfirm({
|
||||
content: '确认提交生成任务？该任务无法终止！'
|
||||
});
|
||||
const { toast } = useToast();
|
||||
const { setIsLoading, Loading } = useLoading();
|
||||
const { File, onOpen } = useSelectFile({ fileType: '.txt,.doc,.docx,.pdf', multiple: true });
|
||||
const { tabs, activeTab, setActiveTab } = useTabs({
|
||||
tabs: [
|
||||
{ id: 'text', label: '文本' },
|
||||
{ id: 'doc', label: '文件' },
|
||||
{ id: 'url', label: '链接' }
|
||||
]
|
||||
});
|
||||
|
||||
const [textInput, setTextInput] = useState('');
|
||||
const [fileText, setFileText] = useState('');
|
||||
|
||||
const { mutate: handleClickSubmit, isLoading } = useMutation({
|
||||
mutationFn: async () => {
|
||||
let text = '';
|
||||
if (activeTab === 'text') {
|
||||
text = textInput;
|
||||
} else if (activeTab === 'doc') {
|
||||
text = fileText;
|
||||
} else if (activeTab === 'url') {
|
||||
}
|
||||
if (!text) return;
|
||||
return postSplitData(dataId, text);
|
||||
},
|
||||
onSuccess() {
|
||||
toast({
|
||||
title: '任务提交成功',
|
||||
status: 'success'
|
||||
});
|
||||
onClose();
|
||||
},
|
||||
onError(err: any) {
|
||||
toast({
|
||||
title: err?.message || '提交任务异常',
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const onSelectFile = useCallback(
|
||||
async (e: File[]) => {
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const fileTexts = (
|
||||
await Promise.all(
|
||||
e.map((file) => {
|
||||
// @ts-ignore
|
||||
const extension = file?.name?.split('.').pop().toLowerCase();
|
||||
if (extension === 'txt') {
|
||||
return readTxtContent(file);
|
||||
} else if (extension === 'pdf') {
|
||||
return readPdfContent(file);
|
||||
} else if (extension === 'docx' || extension === 'doc') {
|
||||
return readDocContent(file);
|
||||
}
|
||||
return '';
|
||||
})
|
||||
)
|
||||
).join('\n');
|
||||
setFileText(fileTexts);
|
||||
} catch (error: any) {
|
||||
console.log(error);
|
||||
toast({
|
||||
title: typeof error === 'string' ? error : '解析文件失败',
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
setIsLoading(false);
|
||||
},
|
||||
[setIsLoading, toast]
|
||||
);
|
||||
|
||||
return (
|
||||
<Modal isOpen={true} onClose={onClose}>
|
||||
<ModalOverlay />
|
||||
<ModalContent position={'relative'} maxW={['90vw', '800px']}>
|
||||
<ModalHeader>导入数据，生成QA</ModalHeader>
|
||||
<ModalCloseButton />
|
||||
|
||||
<ModalBody display={'flex'}>
|
||||
<Box>
|
||||
{tabs.map((item) => (
|
||||
<Button
|
||||
key={item.id}
|
||||
display={'block'}
|
||||
variant={activeTab === item.id ? 'solid' : 'outline'}
|
||||
_notLast={{
|
||||
mb: 3
|
||||
}}
|
||||
onClick={() => setActiveTab(item.id)}
|
||||
>
|
||||
{item.label}
|
||||
</Button>
|
||||
))}
|
||||
</Box>
|
||||
|
||||
<Box flex={'1 0 0'} w={0} ml={3} minH={'200px'}>
|
||||
{activeTab === 'text' && (
|
||||
<Textarea
|
||||
h={'100%'}
|
||||
maxLength={-1}
|
||||
value={textInput}
|
||||
placeholder={'请粘贴或输入需要处理的文本'}
|
||||
onChange={(e) => setTextInput(e.target.value)}
|
||||
/>
|
||||
)}
|
||||
{activeTab === 'doc' && (
|
||||
<Flex
|
||||
flexDirection={'column'}
|
||||
h={'100%'}
|
||||
alignItems={'center'}
|
||||
justifyContent={'center'}
|
||||
border={'1px solid '}
|
||||
borderColor={'blackAlpha.200'}
|
||||
borderRadius={'md'}
|
||||
>
|
||||
<Button onClick={onOpen}>选择文件</Button>
|
||||
{fileText && <Box mt={2}>一共 {fileText.length} 个字</Box>}
|
||||
</Flex>
|
||||
)}
|
||||
</Box>
|
||||
</ModalBody>
|
||||
<ModalFooter>
|
||||
<Button colorScheme={'gray'} onClick={onClose}>
|
||||
取消
|
||||
</Button>
|
||||
<Button
|
||||
ml={3}
|
||||
isLoading={isLoading}
|
||||
isDisabled={!textInput && !fileText}
|
||||
onClick={openConfirm(handleClickSubmit)}
|
||||
>
|
||||
确认
|
||||
</Button>
|
||||
</ModalFooter>
|
||||
<Loading />
|
||||
</ModalContent>
|
||||
|
||||
<ConfirmChild />
|
||||
<File onSelect={onSelectFile} />
|
||||
</Modal>
|
||||
);
|
||||
};
|
||||
|
||||
export default ImportDataModal;
|
||||
111
src/pages/data/list.tsx
Normal file
111
src/pages/data/list.tsx
Normal file
@@ -0,0 +1,111 @@
|
||||
import React, { useState } from 'react';
|
||||
import {
|
||||
Card,
|
||||
Box,
|
||||
Flex,
|
||||
Button,
|
||||
Table,
|
||||
Thead,
|
||||
Tbody,
|
||||
Tr,
|
||||
Th,
|
||||
Td,
|
||||
TableContainer,
|
||||
useDisclosure
|
||||
} from '@chakra-ui/react';
|
||||
import { getDataList } from '@/api/data';
|
||||
import { usePaging } from '@/hooks/usePaging';
|
||||
import type { DataListItem } from '@/types/data';
|
||||
import ScrollData from '@/components/ScrollData';
|
||||
import dayjs from 'dayjs';
|
||||
import dynamic from 'next/dynamic';
|
||||
|
||||
const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
|
||||
const ImportDataModal = dynamic(() => import('./components/ImportDataModal'));
|
||||
|
||||
const DataList = () => {
|
||||
const {
|
||||
setPageNum,
|
||||
pageNum,
|
||||
data: dataList,
|
||||
getData
|
||||
} = usePaging<DataListItem>({
|
||||
api: getDataList,
|
||||
pageSize: 20
|
||||
});
|
||||
const [ImportDataId, setImportDataId] = useState<string>();
|
||||
|
||||
const {
|
||||
isOpen: isOpenCreateDataModal,
|
||||
onOpen: onOpenCreateDataModal,
|
||||
onClose: onCloseCreateDataModal
|
||||
} = useDisclosure();
|
||||
|
||||
return (
|
||||
<Box display={['block', 'flex']} flexDirection={'column'} h={'100%'}>
|
||||
<Card px={6} py={4}>
|
||||
<Flex>
|
||||
<Box flex={1} mr={1}>
|
||||
<Box fontSize={'xl'} fontWeight={'bold'}>
|
||||
对话数据管理
|
||||
</Box>
|
||||
<Box fontSize={'xs'} color={'blackAlpha.600'}>
|
||||
允许你将任意文本数据拆分成 QA 的形式。你可以使用这些 QA 去微调你的对话模型。
|
||||
</Box>
|
||||
</Box>
|
||||
<Button variant={'outline'} onClick={onOpenCreateDataModal}>
|
||||
创建数据集
|
||||
</Button>
|
||||
</Flex>
|
||||
</Card>
|
||||
{/* 数据表 */}
|
||||
<Card mt={3} flex={'1 0 0'} h={['auto', '0']} px={6} py={4}>
|
||||
<ScrollData h={'100%'} nextPage={() => setPageNum(pageNum + 1)}>
|
||||
<TableContainer>
|
||||
<Table>
|
||||
<Thead>
|
||||
<Tr>
|
||||
<Th>集合名</Th>
|
||||
<Th>创建时间</Th>
|
||||
<Th>训练中 / 总数据</Th>
|
||||
<Th></Th>
|
||||
</Tr>
|
||||
</Thead>
|
||||
<Tbody>
|
||||
{dataList.map((item, i) => (
|
||||
<Tr key={item._id}>
|
||||
<Td>{item.name}</Td>
|
||||
<Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
|
||||
<Td>
|
||||
{item.trainingData} / {item.totalData}
|
||||
</Td>
|
||||
<Td>
|
||||
<Button
|
||||
size={'sm'}
|
||||
variant={'outline'}
|
||||
mr={2}
|
||||
onClick={() => setImportDataId(item._id)}
|
||||
>
|
||||
导入
|
||||
</Button>
|
||||
<Button size={'sm'}>导出</Button>
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
</Tbody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
</ScrollData>
|
||||
</Card>
|
||||
|
||||
{ImportDataId && (
|
||||
<ImportDataModal dataId={ImportDataId} onClose={() => setImportDataId(undefined)} />
|
||||
)}
|
||||
{isOpenCreateDataModal && (
|
||||
<CreateDataModal onClose={onCloseCreateDataModal} onSuccess={() => getData(1, true)} />
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default DataList;
|
||||
Reference in New Issue
Block a user