feat: 摘要拆分

src/pages/data/components/CreateDataModal.tsx

@@ -8,10 +8,21 @@ import {
   ModalBody,
   ModalCloseButton,
   Button,
-  Input
+  Input,
+  Select,
+  FormControl,
+  FormErrorMessage
 } from '@chakra-ui/react';
 import { postData } from '@/api/data';
 import { useMutation } from '@tanstack/react-query';
+import { useForm, SubmitHandler } from 'react-hook-form';
+import { DataType } from '@/types/data';
+import { DataTypeTextMap } from '@/constants/data';
+
+export interface CreateDataProps {
+  name: string;
+  type: DataType;
+}
 
 const CreateDataModal = ({
   onClose,
@@ -21,9 +32,20 @@ const CreateDataModal = ({
   onSuccess: () => void;
 }) => {
   const [inputVal, setInputVal] = useState('');
+  const {
+    getValues,
+    register,
+    handleSubmit,
+    formState: { errors }
+  } = useForm<CreateDataProps>({
+    defaultValues: {
+      name: '',
+      type: 'abstract'
+    }
+  });
 
   const { isLoading, mutate } = useMutation({
-    mutationFn: (name: string) => postData(name),
+    mutationFn: (e: CreateDataProps) => postData(e),
     onSuccess() {
       onSuccess();
       onClose();
@@ -37,23 +59,33 @@ const CreateDataModal = ({
         <ModalHeader>创建数据集</ModalHeader>
         <ModalCloseButton />
 
-        <ModalBody display={'flex'}>
-          <Input
-            value={inputVal}
-            onChange={(e) => setInputVal(e.target.value)}
-            placeholder={'数据集名称'}
-          ></Input>
+        <ModalBody>
+          <FormControl mb={8} isInvalid={!!errors.name}>
+            <Input
+              placeholder="数据集名称"
+              {...register('name', {
+                required: '数据集名称不能为空'
+              })}
+            />
+            <FormErrorMessage position={'absolute'} fontSize="xs">
+              {!!errors.name && errors.name.message}
+            </FormErrorMessage>
+          </FormControl>
+          <FormControl>
+            <Select placeholder="数据集类型" {...register('type', {})}>
+              {Object.entries(DataTypeTextMap).map(([key, value]) => (
+                <option key={key} value={key}>
+                  {value}
+                </option>
+              ))}
+            </Select>
+          </FormControl>
         </ModalBody>
         <ModalFooter>
           <Button colorScheme={'gray'} onClick={onClose}>
             取消
           </Button>
-          <Button
-            ml={3}
-            isDisabled={inputVal === ''}
-            isLoading={isLoading}
-            onClick={() => mutate(inputVal)}
-          >
+          <Button ml={3} isLoading={isLoading} onClick={handleSubmit(mutate as any)}>
             确认
           </Button>
         </ModalFooter>

src/pages/data/components/ImportDataModal.tsx

@@ -22,6 +22,7 @@ import { useToast } from '@/hooks/useToast';
 import { useLoading } from '@/hooks/useLoading';
 import { formatPrice } from '@/utils/user';
 import { modelList, ChatModelNameEnum } from '@/constants/model';
+import { encode, decode } from 'gpt-token-utils';
 
 const fileExtension = '.txt,.doc,.docx,.pdf,.md';
 
@@ -106,6 +107,7 @@ const ImportDataModal = ({
           .join('\n')
           .replace(/\n+/g, '\n');
         setFileText(fileTexts);
+        console.log(encode(fileTexts));
       } catch (error: any) {
         console.log(error);
         toast({
@@ -161,7 +163,9 @@ const ImportDataModal = ({
                   placeholder={'请粘贴或输入需要处理的文本'}
                   onChange={(e) => setTextInput(e.target.value)}
                 />
-                <Box mt={2}>一共 {textInput.length} 个字</Box>
+                <Box mt={2}>
+                  一共 {textInput.length} 个字，{encode(textInput).length} 个tokens
+                </Box>
               </>
             )}
             {activeTab === 'doc' && (
@@ -174,12 +178,15 @@ const ImportDataModal = ({
                 border={'1px solid '}
                 borderColor={'blackAlpha.200'}
                 borderRadius={'md'}
+                fontSize={'sm'}
               >
                 <Button onClick={onOpen}>选择文件</Button>
                 <Box mt={2}>支持 {fileExtension} 文件</Box>
                 {fileText && (
                   <>
-                    <Box mt={2}>一共 {fileText.length} 个字</Box>
+                    <Box mt={2}>
+                      一共 {fileText.length} 个字，{encode(fileText).length} 个tokens
+                    </Box>
                     <Box
                       maxH={'300px'}
                       w={'100%'}

src/pages/data/detail.tsx

@@ -22,7 +22,7 @@ const DataDetail = ({ dataName, dataId }: { dataName: string; dataId: string })
   return (
     <Card py={4} h={'100%'} display={'flex'} flexDirection={'column'}>
       <Box px={6} fontSize={'xl'} fontWeight={'bold'}>
-        {dataName} 拆分结果
+        {dataName} 结果
       </Box>
       <ScrollData
         flex={'1 0 0'}
@@ -38,8 +38,13 @@ const DataDetail = ({ dataName, dataId }: { dataName: string; dataId: string })
           <Box key={item._id}>
             {item.result.map((result, i) => (
               <Box key={i} mb={3}>
-                <Box fontWeight={'bold'}>Q: {result.q}</Box>
-                <Box>A: {result.a}</Box>
+                {item.type === 'QA' && (
+                  <>
+                    <Box fontWeight={'bold'}>Q: {result.q}</Box>
+                    <Box>A: {result.a}</Box>
+                  </>
+                )}
+                {item.type === 'abstract' && <Box fontSize={'sm'}>{result.abstract}</Box>}
               </Box>
             ))}
           </Box>

src/pages/data/list.tsx

@@ -28,13 +28,14 @@ import { useRouter } from 'next/router';
 import { useConfirm } from '@/hooks/useConfirm';
 import { useRequest } from '@/hooks/useRequest';
 import { DataItemSchema } from '@/types/mongoSchema';
+import { DataTypeTextMap } from '@/constants/data';
 import { customAlphabet } from 'nanoid';
 const nanoid = customAlphabet('.,', 1);
 
 const CreateDataModal = dynamic(() => import('./components/CreateDataModal'));
 const ImportDataModal = dynamic(() => import('./components/ImportDataModal'));
 
-export type ExportDataType = 'jsonl';
+export type ExportDataType = 'jsonl' | 'txt';
 
 const DataList = () => {
   const router = useRouter();
@@ -84,21 +85,26 @@ const DataList = () => {
       let text = '';
       // 生成 jsonl
       data.forEach((item) => {
-        const result = JSON.stringify({
-          prompt: `${item.q.toLocaleLowerCase()}${nanoid()}</s>`,
-          completion: ` ${item.a}###`
-        });
-        text += `${result}\n`;
+        if (res.type === 'jsonl' && item.q && item.a) {
+          const result = JSON.stringify({
+            prompt: `${item.q.toLocaleLowerCase()}${nanoid()}</s>`,
+            completion: ` ${item.a}###`
+          });
+          text += `${result}\n`;
+        } else if (res.type === 'txt' && item.abstract) {
+          text += `${item.abstract}\n`;
+        }
       });
       // 去掉最后一个 \n
       text = text.substring(0, text.length - 1);
+
       // 导出为文件
       const blob = new Blob([text], { type: 'application/json;charset=utf-8' });
 
       // 创建下载链接
       const downloadLink = document.createElement('a');
       downloadLink.href = window.URL.createObjectURL(blob);
-      downloadLink.download = 'file.jsonl';
+      downloadLink.download = `data.${res.type}`;
 
       // 添加链接到页面并触发下载
       document.body.appendChild(downloadLink);
@@ -138,6 +144,7 @@ const DataList = () => {
               <Thead>
                 <Tr>
                   <Th>集合名</Th>
+                  <Th>类型</Th>
                   <Th>创建时间</Th>
                   <Th>训练中 / 总数据</Th>
                   <Th></Th>
@@ -158,6 +165,7 @@ const DataList = () => {
                         }}
                       />
                     </Td>
+                    <Td>{DataTypeTextMap[item.type || 'QA']}</Td>
                     <Td>{dayjs(item.createTime).format('YYYY/MM/DD HH:mm')}</Td>
                     <Td>
                       {item.trainingData} / {item.totalData}
@@ -187,9 +195,18 @@ const DataList = () => {
                           导出
                         </MenuButton>
                         <MenuList>
-                          <MenuItem onClick={() => handleExportData({ data: item, type: 'jsonl' })}>
-                            jsonl
-                          </MenuItem>
+                          {item.type === 'QA' && (
+                            <MenuItem
+                              onClick={() => handleExportData({ data: item, type: 'jsonl' })}
+                            >
+                              jsonl
+                            </MenuItem>
+                          )}
+                          {item.type === 'abstract' && (
+                            <MenuItem onClick={() => handleExportData({ data: item, type: 'txt' })}>
+                              txt
+                            </MenuItem>
+                          )}
                         </MenuList>
                       </Menu>