perf: 知识库数据结构

2023-04-01 22:31:56 +08:00
parent 5759cbeae0
commit ae4243b522
26 changed files with 611 additions and 518 deletions
--- a/src/pages/model/detail/components/InputDataModal.tsx
+++ b/src/pages/model/detail/components/InputDataModal.tsx
@@ -45,24 +45,22 @@ const InputDataModal = ({
      setImporting(true);

      try {
-        await postModelDataInput({
+        const res = await postModelDataInput({
          modelId: modelId,
          data: [
            {
              text: e.text,
-              q: [
-                {
-                  id: nanoid(),
-                  text: e.q
-                }
-              ]
+              q: {
+                id: nanoid(),
+                text: e.q
+              }
            }
          ]
        });

        toast({
-          title: '导入数据成功,需要一段时间训练',
-          status: 'success'
+          title: res === 0 ? '导入数据成功,需要一段时间训练' : '数据导入异常',
+          status: res === 0 ? 'success' : 'warning'
        });
        onClose();
        onSuccess();
@@ -88,8 +86,15 @@ const InputDataModal = ({
        <ModalHeader>手动导入</ModalHeader>
        <ModalCloseButton />

-        <Flex flex={'1 0 0'} h={0} px={6} pb={2}>
-          <Box flex={2} mr={4} h={'100%'}>
+        <Box
+          display={['block', 'flex']}
+          flex={'1 0 0'}
+          h={['100%', 0]}
+          overflowY={'auto'}
+          px={6}
+          pb={2}
+        >
+          <Box flex={2} mr={[0, 4]} mb={[4, 0]} h={['230px', '100%']}>
            <Box h={'30px'}>问题</Box>
            <Textarea
              placeholder="相关问题，可以回车输入多个问法, 最多500字"
@@ -101,10 +106,11 @@ const InputDataModal = ({
              })}
            />
          </Box>
-          <Box flex={3} h={'100%'}>
+          <Box flex={3} h={['330px', '100%']}>
            <Box h={'30px'}>知识点</Box>
            <Textarea
-              placeholder="知识点"
+              placeholder="知识点,最多1000字"
+              maxLength={1000}
              resize={'none'}
              h={'calc(100% - 30px)'}
              {...register(`text`, {
@@ -112,7 +118,7 @@ const InputDataModal = ({
              })}
            />
          </Box>
-        </Flex>
+        </Box>

        <Flex px={6} pt={2} pb={4}>
          <Box flex={1}></Box>
--- a/src/pages/model/detail/components/ModelDataCard.tsx
+++ b/src/pages/model/detail/components/ModelDataCard.tsx
@@ -19,7 +19,7 @@ import {
  MenuItem
 } from '@chakra-ui/react';
 import type { ModelSchema } from '@/types/mongoSchema';
-import { ModelDataSchema } from '@/types/mongoSchema';
+import type { RedisModelDataItemType } from '@/types/redis';
 import { ModelDataStatusMap } from '@/constants/model';
 import { usePagination } from '@/hooks/usePagination';
 import {
@@ -35,7 +35,8 @@ import dynamic from 'next/dynamic';
 import { useQuery } from '@tanstack/react-query';

 const InputModel = dynamic(() => import('./InputDataModal'));
-const SelectModel = dynamic(() => import('./SelectFileModal'));
+const SelectFileModel = dynamic(() => import('./SelectFileModal'));
+const SelectJsonModel = dynamic(() => import('./SelectJsonModal'));

 const ModelDataCard = ({ model }: { model: ModelSchema }) => {
  const { toast } = useToast();
@@ -48,7 +49,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
    total,
    getData,
    pageNum
-  } = usePagination<ModelDataSchema>({
+  } = usePagination<RedisModelDataItemType>({
    api: getModelDataList,
    pageSize: 8,
    params: {
@@ -76,12 +77,17 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
    onClose: onCloseInputModal
  } = useDisclosure();
  const {
-    isOpen: isOpenSelectModal,
-    onOpen: onOpenSelectModal,
-    onClose: onCloseSelectModal
+    isOpen: isOpenSelectFileModal,
+    onOpen: onOpenSelectFileModal,
+    onClose: onCloseSelectFileModal
+  } = useDisclosure();
+  const {
+    isOpen: isOpenSelectJsonModal,
+    onOpen: onOpenSelectJsonModal,
+    onClose: onCloseSelectJsonModal
  } = useDisclosure();

-  const { data, refetch } = useQuery(['getModelSplitDataList'], () =>
+  const { data: splitDataList, refetch } = useQuery(['getModelSplitDataList'], () =>
    getModelSplitDataList(model._id)
  );

@@ -113,13 +119,18 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
          <MenuButton as={Button}>导入</MenuButton>
          <MenuList>
            <MenuItem onClick={onOpenInputModal}>手动输入</MenuItem>
-            <MenuItem onClick={onOpenSelectModal}>文件导入</MenuItem>
+            <MenuItem onClick={onOpenSelectFileModal}>文件导入</MenuItem>
+            <MenuItem onClick={onOpenSelectJsonModal}>JSON导入</MenuItem>
          </MenuList>
        </Menu>
      </Flex>
-      {data && data.length > 0 && <Box fontSize={'xs'}>{data.length}条数据正在拆分中...</Box>}
+      {splitDataList && splitDataList.length > 0 && (
+        <Box fontSize={'xs'}>
+          {splitDataList.map((item) => item.textList).flat().length}条数据正在拆分...
+        </Box>
+      )}
      <Box mt={4}>
-        <TableContainer>
+        <TableContainer minH={'500px'}>
          <Table variant={'simple'}>
            <Thead>
              <Tr>
@@ -131,19 +142,11 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
            </Thead>
            <Tbody>
              {modelDataList.map((item) => (
-                <Tr key={item._id}>
+                <Tr key={item.id}>
                  <Td w={'350px'}>
-                    {item.q.map((item, i) => (
-                      <Box
-                        key={item.id}
-                        fontSize={'xs'}
-                        w={'100%'}
-                        whiteSpace={'pre-wrap'}
-                        _notLast={{ mb: 1 }}
-                      >
-                        {item.text}
-                      </Box>
-                    ))}
+                    <Box fontSize={'xs'} w={'100%'} whiteSpace={'pre-wrap'} _notLast={{ mb: 1 }}>
+                      {item.q}
+                    </Box>
                  </Td>
                  <Td minW={'200px'}>
                    <Textarea
@@ -153,9 +156,9 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
                      fontSize={'xs'}
                      resize={'both'}
                      onBlur={(e) => {
-                        const oldVal = modelDataList.find((data) => item._id === data._id)?.text;
+                        const oldVal = modelDataList.find((data) => item.id === data.id)?.text;
                        if (oldVal !== e.target.value) {
-                          updateAnswer(item._id, e.target.value);
+                          updateAnswer(item.id, e.target.value);
                        }
                      }}
                    ></Textarea>
@@ -169,7 +172,7 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
                      aria-label={'delete'}
                      size={'sm'}
                      onClick={async () => {
-                        await delOneModelData(item._id);
+                        await delOneModelData(item.id);
                        refetchData(pageNum);
                      }}
                    />
@@ -188,8 +191,19 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
      {isOpenInputModal && (
        <InputModel modelId={model._id} onClose={onCloseInputModal} onSuccess={refetchData} />
      )}
-      {isOpenSelectModal && (
-        <SelectModel modelId={model._id} onClose={onCloseSelectModal} onSuccess={refetchData} />
+      {isOpenSelectFileModal && (
+        <SelectFileModel
+          modelId={model._id}
+          onClose={onCloseSelectFileModal}
+          onSuccess={refetchData}
+        />
+      )}
+      {isOpenSelectJsonModal && (
+        <SelectJsonModel
+          modelId={model._id}
+          onClose={onCloseSelectJsonModal}
+          onSuccess={refetchData}
+        />
      )}
    </>
  );
--- a/src/pages/model/detail/components/SelectFileModal.tsx
+++ b/src/pages/model/detail/components/SelectFileModal.tsx
@@ -100,40 +100,43 @@ const SelectFileModal = ({
  });

  return (
-    <Modal isOpen={true} onClose={onClose}>
+    <Modal isOpen={true} onClose={onClose} isCentered>
      <ModalOverlay />
-      <ModalContent maxW={'min(900px, 90vw)'} position={'relative'}>
+      <ModalContent maxW={'min(900px, 90vw)'} m={0} position={'relative'} h={['90vh', '70vh']}>
        <ModalHeader>文件导入</ModalHeader>
        <ModalCloseButton />

-        <ModalBody>
-          <Flex
-            flexDirection={'column'}
+        <ModalBody
+          display={'flex'}
+          flexDirection={'column'}
+          p={4}
+          h={'100%'}
+          alignItems={'center'}
+          justifyContent={'center'}
+          fontSize={'sm'}
+        >
+          <Button isLoading={selecting} onClick={onOpen}>
+            选择文件
+          </Button>
+          <Box mt={2} maxW={['100%', '70%']}>
+            支持 {fileExtension} 文件。模型会自动对文本进行 QA 拆分，需要较长训练时间，拆分需要消耗
+            tokens，大约0.04元/1k tokens，请确保账号余额充足。
+          </Box>
+          <Box mt={2}>
+            一共 {fileText.length} 个字，{encode(fileText).length} 个tokens
+          </Box>
+          <Box
+            flex={'1 0 0'}
+            h={0}
+            w={'100%'}
+            overflowY={'auto'}
            p={2}
-            h={'100%'}
-            alignItems={'center'}
-            justifyContent={'center'}
-            fontSize={'sm'}
+            backgroundColor={'blackAlpha.50'}
+            whiteSpace={'pre-wrap'}
+            fontSize={'xs'}
          >
-            <Button isLoading={selecting} onClick={onOpen}>
-              选择文件
-            </Button>
-            <Box mt={2}>支持 {fileExtension} 文件. 会先对文本进行拆分，需要时间较长。</Box>
-            <Box mt={2}>
-              一共 {fileText.length} 个字，{encode(fileText).length} 个tokens
-            </Box>
-            <Box
-              h={'300px'}
-              w={'100%'}
-              overflow={'auto'}
-              p={2}
-              backgroundColor={'blackAlpha.50'}
-              whiteSpace={'pre'}
-              fontSize={'xs'}
-            >
-              {fileText}
-            </Box>
-          </Flex>
+            {fileText}
+          </Box>
        </ModalBody>

        <Flex px={6} pt={2} pb={4}>
--- a/src/pages/model/detail/components/SelectJsonModal.tsx
+++ b/src/pages/model/detail/components/SelectJsonModal.tsx
@@ -0,0 +1,145 @@
+import React, { useState, useCallback } from 'react';
+import {
+  Box,
+  Flex,
+  Button,
+  Modal,
+  ModalOverlay,
+  ModalContent,
+  ModalHeader,
+  ModalCloseButton,
+  ModalBody
+} from '@chakra-ui/react';
+import { useToast } from '@/hooks/useToast';
+import { useSelectFile } from '@/hooks/useSelectFile';
+import { useConfirm } from '@/hooks/useConfirm';
+import { readTxtContent } from '@/utils/tools';
+import { useMutation } from '@tanstack/react-query';
+import { postModelDataJsonData } from '@/api/model';
+import Markdown from '@/components/Markdown';
+
+const SelectJsonModal = ({
+  onClose,
+  onSuccess,
+  modelId
+}: {
+  onClose: () => void;
+  onSuccess: () => void;
+  modelId: string;
+}) => {
+  const [selecting, setSelecting] = useState(false);
+  const { toast } = useToast();
+  const { File, onOpen } = useSelectFile({ fileType: '.json', multiple: true });
+  const [fileData, setFileData] = useState<
+    { prompt: string; completion: string; vector?: number[] }[]
+  >([]);
+  const { openConfirm, ConfirmChild } = useConfirm({
+    content: '确认导入该数据集?'
+  });
+
+  const onSelectFile = useCallback(
+    async (e: File[]) => {
+      setSelecting(true);
+      try {
+        const jsonData = (
+          await Promise.all(e.map((item) => readTxtContent(item).then((text) => JSON.parse(text))))
+        ).flat();
+        // check 文件类型
+        for (let i = 0; i < jsonData.length; i++) {
+          if (!jsonData[i]?.prompt || !jsonData[i]?.completion) {
+            throw new Error('缺少 prompt 或 completion');
+          }
+        }
+
+        setFileData(jsonData);
+      } catch (error: any) {
+        console.log(error);
+        toast({
+          title: error?.message || 'JSON文件格式有误',
+          status: 'error'
+        });
+      }
+      setSelecting(false);
+    },
+    [setSelecting, toast]
+  );
+
+  const { mutate, isLoading } = useMutation({
+    mutationFn: async () => {
+      if (!fileData) return;
+      const res = await postModelDataJsonData(modelId, fileData);
+      console.log(res);
+      toast({
+        title: '导入数据成功,需要一段拆解和训练',
+        status: 'success'
+      });
+      onClose();
+      onSuccess();
+    },
+    onError() {
+      toast({
+        title: '导入文件失败',
+        status: 'error'
+      });
+    }
+  });
+
+  return (
+    <Modal isOpen={true} onClose={onClose} isCentered>
+      <ModalOverlay />
+      <ModalContent maxW={'90vw'} position={'relative'} m={0} h={'90vh'}>
+        <ModalHeader>JSON数据集</ModalHeader>
+        <ModalCloseButton />
+
+        <ModalBody h={'100%'} display={['block', 'flex']} fontSize={'sm'} overflowY={'auto'}>
+          <Box flex={'2 0 0'} w={['100%', 0]} mr={[0, 4]} mb={[4, 0]}>
+            <Markdown
+              source={`接受一个对象数组，每个对象必须包含 prompt 和 completion 格式，可以包含vector。prompt 代表问题，completion 代表回答的内容，可以多个问题对应一个回答，vector 为 prompt 的向量，如果没有讲有系统生成。例如：
+~~~json
+[
+  {
+    "prompt":"sealos是什么?\\n介绍下sealos\\nsealos有什么用",
+    "completion":"sealos是xxxxxx"
+  },
+  {
+    "prompt":"laf是什么?",
+    "completion":"laf是xxxxxx",
+    "vector":[-0.42,-0.4314314,0.43143]
+  }
+]
+~~~`}
+            />
+            <Flex alignItems={'center'}>
+              <Button isLoading={selecting} onClick={onOpen}>
+                选择 JSON 数据集
+              </Button>
+
+              <Box ml={4}>一共 {fileData.length} 组数据</Box>
+            </Flex>
+          </Box>
+          <Box flex={'2 0 0'} h={'100%'} overflow={'auto'} p={2} backgroundColor={'blackAlpha.50'}>
+            {JSON.stringify(fileData)}
+          </Box>
+        </ModalBody>
+
+        <Flex px={6} pt={2} pb={4}>
+          <Box flex={1}></Box>
+          <Button variant={'outline'} mr={3} onClick={onClose}>
+            取消
+          </Button>
+          <Button
+            isLoading={isLoading}
+            isDisabled={fileData.length === 0}
+            onClick={openConfirm(mutate)}
+          >
+            确认导入
+          </Button>
+        </Flex>
+      </ModalContent>
+      <ConfirmChild />
+      <File onSelect={onSelectFile} />
+    </Modal>
+  );
+};
+
+export default SelectJsonModal;