feat: kb ui

This commit is contained in:
archer
2023-07-14 11:49:36 +08:00
parent 358c4716f9
commit 5a96e167ee
24 changed files with 463 additions and 412 deletions

View File

@@ -0,0 +1,317 @@
import React, { useCallback, useState, useRef, useEffect } from 'react';
import {
Box,
Card,
IconButton,
Flex,
Button,
useDisclosure,
Menu,
MenuButton,
MenuList,
MenuItem,
Input,
Grid
} from '@chakra-ui/react';
import type { KbDataItemType } from '@/types/plugin';
import { usePagination } from '@/hooks/usePagination';
import {
getKbDataList,
getExportDataList,
delOneKbDataByDataId,
getTrainingData
} from '@/api/plugins/kb';
import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons';
import { fileDownload } from '@/utils/file';
import { useMutation, useQuery } from '@tanstack/react-query';
import { useToast } from '@/hooks/useToast';
import Papa from 'papaparse';
import dynamic from 'next/dynamic';
import InputModal, { FormData as InputDataType } from './InputDataModal';
import { debounce } from 'lodash';
import { getErrText } from '@/utils/tools';
const SelectFileModal = dynamic(() => import('./SelectFileModal'), { ssr: true });
const SelectCsvModal = dynamic(() => import('./SelectCsvModal'), { ssr: true });
const DataCard = ({ kbId }: { kbId: string }) => {
const lastSearch = useRef('');
const [searchText, setSearchText] = useState('');
const { toast } = useToast();
const [isDeleting, setIsDeleting] = useState(false);
const {
data: kbDataList,
isLoading,
Pagination,
total,
getData,
pageNum
} = usePagination<KbDataItemType>({
api: getKbDataList,
pageSize: 24,
defaultRequest: false,
params: {
kbId,
searchText
}
});
const [editInputData, setEditInputData] = useState<InputDataType>();
const {
isOpen: isOpenSelectFileModal,
onOpen: onOpenSelectFileModal,
onClose: onCloseSelectFileModal
} = useDisclosure();
const {
isOpen: isOpenSelectCsvModal,
onOpen: onOpenSelectCsvModal,
onClose: onCloseSelectCsvModal
} = useDisclosure();
const { data: { qaListLen = 0, vectorListLen = 0 } = {}, refetch } = useQuery(
['getModelSplitDataList', kbId],
() => getTrainingData({ kbId, init: false }),
{
onError(err) {
console.log(err);
}
}
);
const refetchData = useCallback(
(num = pageNum) => {
getData(num);
refetch();
return null;
},
[getData, pageNum, refetch]
);
// get al data and export csv
const { mutate: onclickExport, isLoading: isLoadingExport = false } = useMutation({
mutationFn: () => getExportDataList(kbId),
onSuccess(res) {
try {
const text = Papa.unparse({
fields: ['question', 'answer', 'source'],
data: res
});
fileDownload({
text,
type: 'text/csv',
filename: 'data.csv'
});
toast({
title: '导出成功,下次导出需要半小时后',
status: 'success'
});
} catch (error) {
error;
}
},
onError(err: any) {
toast({
title: typeof err === 'string' ? err : err?.message || '导出异常',
status: 'error'
});
console.log(err);
}
});
const getFirstData = useCallback(
debounce(() => {
getData(1);
lastSearch.current = searchText;
}, 300),
[]
);
// interval get data
useQuery(['refetchData'], () => refetchData(1), {
refetchInterval: 5000,
enabled: qaListLen > 0 || vectorListLen > 0
});
useEffect(() => {
setSearchText('');
getData(1);
}, [kbId]);
return (
<Box position={'relative'} px={5} py={[1, 5]}>
<Flex justifyContent={'space-between'}>
<Box fontWeight={'bold'} fontSize={'lg'} mr={2}>
: {total}
</Box>
<Box>
<IconButton
icon={<RepeatIcon />}
aria-label={'refresh'}
variant={'base'}
isLoading={isLoading}
mr={[2, 4]}
size={'sm'}
onClick={() => {
getData(pageNum);
getTrainingData({ kbId, init: true });
}}
/>
<Button
variant={'base'}
mr={2}
size={'sm'}
isLoading={isLoadingExport || isLoading}
title={'半小时仅能导出1次'}
onClick={() => onclickExport()}
>
csv
</Button>
<Menu autoSelect={false}>
<MenuButton as={Button} size={'sm'} isLoading={isLoading}>
</MenuButton>
<MenuList>
<MenuItem
onClick={() =>
setEditInputData({
a: '',
q: ''
})
}
>
</MenuItem>
<MenuItem onClick={onOpenSelectFileModal}>/</MenuItem>
<MenuItem onClick={onOpenSelectCsvModal}>csv </MenuItem>
</MenuList>
</Menu>
</Box>
</Flex>
<Flex my={4}>
{qaListLen > 0 || vectorListLen > 0 ? (
<Box fontSize={'xs'}>
{qaListLen > 0 ? `${qaListLen}条数据正在拆分,` : ''}
{vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引,` : ''}
...
</Box>
) : (
<Box fontSize={'xs'}>~</Box>
)}
<Box flex={1} mr={1} />
<Input
maxW={['60%', '300px']}
size={'sm'}
value={searchText}
placeholder="根据匹配知识,补充知识和来源搜索"
onChange={(e) => {
setSearchText(e.target.value);
getFirstData();
}}
onBlur={() => {
if (searchText === lastSearch.current) return;
getFirstData();
}}
onKeyDown={(e) => {
if (searchText === lastSearch.current) return;
if (e.key === 'Enter') {
getFirstData();
}
}}
/>
</Flex>
<Grid
minH={'100px'}
gridTemplateColumns={['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)']}
gridGap={4}
>
{kbDataList.map((item) => (
<Card
key={item.id}
cursor={'pointer'}
pt={3}
userSelect={'none'}
boxShadow={'none'}
_hover={{ boxShadow: 'lg', '& .delete': { display: 'flex' } }}
border={'1px solid '}
borderColor={'myGray.200'}
onClick={() =>
setEditInputData({
dataId: item.id,
q: item.q,
a: item.a
})
}
>
<Box
h={'100px'}
overflow={'hidden'}
wordBreak={'break-all'}
px={3}
py={1}
fontSize={'13px'}
>
<Box color={'myGray.1000'} mb={2}>
{item.q}
</Box>
<Box color={'myGray.600'}>{item.a}</Box>
</Box>
<Flex py={2} px={4} h={'36px'} alignItems={'flex-end'} fontSize={'sm'}>
<Box className={'textEllipsis'} flex={1}>
{item.source?.trim()}
</Box>
<IconButton
className="delete"
display={['flex', 'none']}
icon={<DeleteIcon />}
variant={'base'}
colorScheme={'gray'}
aria-label={'delete'}
size={'xs'}
borderRadius={'md'}
_hover={{ color: 'red.600' }}
isLoading={isDeleting}
onClick={async (e) => {
e.stopPropagation();
try {
setIsDeleting(true);
await delOneKbDataByDataId(item.id);
refetchData(pageNum);
} catch (error) {
toast({
title: getErrText(error),
status: 'error'
});
}
setIsDeleting(false);
}}
/>
</Flex>
</Card>
))}
</Grid>
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
{editInputData !== undefined && (
<InputModal
kbId={kbId}
defaultValues={editInputData}
onClose={() => setEditInputData(undefined)}
onSuccess={() => refetchData()}
/>
)}
{isOpenSelectFileModal && (
<SelectFileModal kbId={kbId} onClose={onCloseSelectFileModal} onSuccess={refetchData} />
)}
{isOpenSelectCsvModal && (
<SelectCsvModal kbId={kbId} onClose={onCloseSelectCsvModal} onSuccess={refetchData} />
)}
</Box>
);
};
export default React.memo(DataCard);

View File

@@ -0,0 +1,233 @@
import React, {
useCallback,
useState,
useRef,
forwardRef,
useImperativeHandle,
ForwardedRef
} from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, Button, FormControl, IconButton, Input, Card } from '@chakra-ui/react';
import { QuestionOutlineIcon, DeleteIcon } from '@chakra-ui/icons';
import { delKbById, putKbById } from '@/api/plugins/kb';
import { useSelectFile } from '@/hooks/useSelectFile';
import { useToast } from '@/hooks/useToast';
import { useUserStore } from '@/store/user';
import { useConfirm } from '@/hooks/useConfirm';
import { UseFormReturn } from 'react-hook-form';
import { compressImg } from '@/utils/file';
import type { KbItemType } from '@/types/plugin';
import Avatar from '@/components/Avatar';
import Tag from '@/components/Tag';
import MyTooltip from '@/components/MyTooltip';
export interface ComponentRef {
initInput: (tags: string) => void;
}
const Info = (
{ kbId, form }: { kbId: string; form: UseFormReturn<KbItemType, any> },
ref: ForwardedRef<ComponentRef>
) => {
const { getValues, formState, setValue, register, handleSubmit } = form;
const InputRef = useRef<HTMLInputElement>(null);
const { toast } = useToast();
const router = useRouter();
const [btnLoading, setBtnLoading] = useState(false);
const [refresh, setRefresh] = useState(false);
const { openConfirm, ConfirmChild } = useConfirm({
content: '确认删除该知识库?数据将无法恢复,请确认!'
});
const { File, onOpen: onOpenSelectFile } = useSelectFile({
fileType: '.jpg,.png',
multiple: false
});
const { kbDetail, getKbDetail, loadKbList, myKbList } = useUserStore();
/* 点击删除 */
const onclickDelKb = useCallback(async () => {
setBtnLoading(true);
try {
await delKbById(kbId);
toast({
title: '删除成功',
status: 'success'
});
router.replace(`/kb?kbId=${myKbList.find((item) => item._id !== kbId)?._id || ''}`);
await loadKbList();
} catch (err: any) {
toast({
title: err?.message || '删除失败',
status: 'error'
});
}
setBtnLoading(false);
}, [setBtnLoading, kbId, toast, router, myKbList, loadKbList]);
const saveSubmitSuccess = useCallback(
async (data: KbItemType) => {
setBtnLoading(true);
try {
await putKbById({
id: kbId,
...data
});
await getKbDetail(kbId, true);
toast({
title: '更新成功',
status: 'success'
});
loadKbList();
} catch (err: any) {
toast({
title: err?.message || '更新失败',
status: 'error'
});
}
setBtnLoading(false);
},
[getKbDetail, kbId, loadKbList, toast]
);
const saveSubmitError = useCallback(() => {
// deep search message
const deepSearch = (obj: any): string => {
if (!obj) return '提交表单错误';
if (!!obj.message) {
return obj.message;
}
return deepSearch(Object.values(obj)[0]);
};
toast({
title: deepSearch(formState.errors),
status: 'error',
duration: 4000,
isClosable: true
});
}, [formState.errors, toast]);
const onSelectFile = useCallback(
async (e: File[]) => {
const file = e[0];
if (!file) return;
try {
const src = await compressImg({
file,
maxW: 100,
maxH: 100
});
setValue('avatar', src);
setRefresh((state) => !state);
} catch (err: any) {
toast({
title: typeof err === 'string' ? err : '头像选择异常',
status: 'warning'
});
}
},
[setRefresh, setValue, toast]
);
useImperativeHandle(ref, () => ({
initInput: (tags: string) => {
console.log(tags);
if (InputRef.current) {
InputRef.current.value = tags;
}
}
}));
return (
<Flex p={5} flexDirection={'column'} alignItems={'center'}>
<Flex mt={5} w={'100%'} maxW={'350px'} alignItems={'center'}>
<Box flex={'0 0 90px'} w={0}>
</Box>
<Box flex={1}>
<Avatar
m={'auto'}
src={getValues('avatar')}
w={['32px', '40px']}
h={['32px', '40px']}
cursor={'pointer'}
title={'点击切换头像'}
onClick={onOpenSelectFile}
/>
</Box>
</Flex>
<FormControl mt={8} w={'100%'} maxW={'350px'} display={'flex'} alignItems={'center'}>
<Box flex={'0 0 90px'} w={0}>
</Box>
<Input
flex={1}
{...register('name', {
required: '知识库名称不能为空'
})}
/>
</FormControl>
<Flex mt={8} alignItems={'center'} w={'100%'} maxW={'350px'} flexWrap={'wrap'}>
<Box flex={'0 0 90px'} w={0}>
<MyTooltip label={'用空格隔开多个标签,便于搜索'}>
<QuestionOutlineIcon ml={1} />
</MyTooltip>
</Box>
<Input
flex={1}
maxW={'300px'}
ref={InputRef}
placeholder={'标签,使用空格分割。'}
maxLength={30}
onChange={(e) => {
setValue('tags', e.target.value);
setRefresh(!refresh);
}}
/>
<Box pl={'90px'} mt={2} w="100%">
{getValues('tags')
.split(' ')
.filter((item) => item)
.map((item, i) => (
<Tag mr={2} mb={2} key={i} whiteSpace={'nowrap'}>
{item}
</Tag>
))}
</Box>
</Flex>
<Flex mt={5} w={'100%'} maxW={'350px'} alignItems={'flex-end'}>
<Box flex={'0 0 90px'} w={0}></Box>
<Button
isLoading={btnLoading}
mr={4}
w={'100px'}
onClick={handleSubmit(saveSubmitSuccess, saveSubmitError)}
>
</Button>
<IconButton
isLoading={btnLoading}
icon={<DeleteIcon />}
aria-label={''}
variant={'outline'}
size={'sm'}
_hover={{
color: 'red.600',
borderColor: 'red.600'
}}
onClick={openConfirm(onclickDelKb)}
/>
</Flex>
<File onSelect={onSelectFile} />
<ConfirmChild />
</Flex>
);
};
export default forwardRef(Info);

View File

@@ -0,0 +1,223 @@
import React, { useState, useCallback } from 'react';
import {
Box,
Flex,
Button,
Modal,
ModalOverlay,
ModalContent,
ModalHeader,
ModalCloseButton,
Textarea,
IconButton
} from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
import { postKbDataFromList, putKbDataById, delOneKbDataByDataId } from '@/api/plugins/kb';
import { useToast } from '@/hooks/useToast';
import { TrainingModeEnum } from '@/constants/plugin';
import { getErrText } from '@/utils/tools';
import MyIcon from '@/components/Icon';
import { vectorModelList } from '@/store/static';
export type FormData = { dataId?: string; a: string; q: string };
const InputDataModal = ({
onClose,
onSuccess,
onDelete,
kbId,
defaultValues = {
a: '',
q: ''
}
}: {
onClose: () => void;
onSuccess: (data: FormData) => void;
onDelete?: () => void;
kbId: string;
defaultValues?: FormData;
}) => {
const [loading, setLoading] = useState(false);
const { toast } = useToast();
const { register, handleSubmit, reset } = useForm<FormData>({
defaultValues
});
/**
* 确认导入新数据
*/
const sureImportData = useCallback(
async (e: FormData) => {
if (e.a.length + e.q.length >= 3000) {
toast({
title: '总长度超长了',
status: 'warning'
});
return;
}
setLoading(true);
try {
const data = {
a: e.a,
q: e.q,
source: '手动录入'
};
const { insertLen } = await postKbDataFromList({
kbId,
model: vectorModelList[0].model,
mode: TrainingModeEnum.index,
data: [data]
});
if (insertLen === 0) {
toast({
title: '已存在完全一致的数据',
status: 'warning'
});
} else {
toast({
title: '导入数据成功,需要一段时间训练',
status: 'success'
});
reset({
a: '',
q: ''
});
}
onSuccess(data);
} catch (err: any) {
toast({
title: getErrText(err, '出现了点意外~'),
status: 'error'
});
}
setLoading(false);
},
[kbId, onSuccess, reset, toast]
);
const updateData = useCallback(
async (e: FormData) => {
if (!e.dataId) return;
if (e.a !== defaultValues.a || e.q !== defaultValues.q) {
setLoading(true);
try {
const data = {
dataId: e.dataId,
a: e.a,
q: e.q === defaultValues.q ? '' : e.q
};
await putKbDataById(data);
onSuccess(data);
} catch (error) {}
setLoading(false);
}
toast({
title: '修改数据成功',
status: 'success'
});
onClose();
},
[defaultValues, onClose, onSuccess, toast]
);
return (
<Modal isOpen={true} onClose={onClose} isCentered>
<ModalOverlay />
<ModalContent
m={0}
display={'flex'}
flexDirection={'column'}
h={'90vh'}
maxW={'90vw'}
position={'relative'}
>
<ModalHeader>{defaultValues.dataId ? '变更数据' : '手动导入数据'}</ModalHeader>
<ModalCloseButton />
<Box
display={'flex'}
flexDirection={['column', 'row']}
flex={'1 0 0'}
h={['100%', 0]}
overflow={'overlay'}
px={6}
pb={2}
>
<Box flex={1} mr={[0, 4]} mb={[4, 0]} h={['50%', '100%']}>
<Box h={'30px'}>{'匹配的知识点'}</Box>
<Textarea
placeholder={'匹配的知识点。这部分内容会被搜索,请把控内容的质量。总和最多 3000 字。'}
maxLength={3000}
resize={'none'}
h={'calc(100% - 30px)'}
{...register(`q`, {
required: true
})}
/>
</Box>
<Box flex={1} h={['50%', '100%']}>
<Box h={'30px'}></Box>
<Textarea
placeholder={
'补充知识。这部分内容不会被搜索,但会作为"匹配的知识点"的内容补充,你可以讲一些细节的内容填写在这里。总和最多 3000 字。'
}
maxLength={3000}
resize={'none'}
h={'calc(100% - 30px)'}
{...register('a')}
/>
</Box>
</Box>
<Flex px={6} pt={2} pb={4} alignItems={'center'}>
<Box flex={1}>
{defaultValues.dataId && onDelete && (
<IconButton
variant={'outline'}
icon={<MyIcon name={'delete'} w={'16px'} h={'16px'} />}
aria-label={''}
isLoading={loading}
size={'sm'}
_hover={{
color: 'red.600',
borderColor: 'red.600'
}}
onClick={async () => {
if (!onDelete || !defaultValues.dataId) return;
try {
await delOneKbDataByDataId(defaultValues.dataId);
onDelete();
onClose();
} catch (error) {
toast({
status: 'warning',
title: getErrText(error)
});
console.log(error);
}
}}
/>
)}
</Box>
<Button variant={'base'} mr={3} isLoading={loading} onClick={onClose}>
</Button>
<Button
isLoading={loading}
onClick={handleSubmit(defaultValues.dataId ? updateData : sureImportData)}
>
{defaultValues.dataId ? '确认变更' : '确认导入'}
</Button>
</Flex>
</ModalContent>
</Modal>
);
};
export default InputDataModal;

View File

@@ -0,0 +1,180 @@
import React, { useState, useCallback } from 'react';
import {
Box,
Flex,
Button,
Modal,
ModalOverlay,
ModalContent,
ModalHeader,
ModalCloseButton,
ModalBody
} from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast';
import { useSelectFile } from '@/hooks/useSelectFile';
import { useConfirm } from '@/hooks/useConfirm';
import { readCsvContent } from '@/utils/file';
import { useMutation } from '@tanstack/react-query';
import { postKbDataFromList } from '@/api/plugins/kb';
import Markdown from '@/components/Markdown';
import { useMarkdown } from '@/hooks/useMarkdown';
import { fileDownload } from '@/utils/file';
import { TrainingModeEnum } from '@/constants/plugin';
import { getErrText } from '@/utils/tools';
const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
const SelectJsonModal = ({
onClose,
onSuccess,
kbId
}: {
onClose: () => void;
onSuccess: () => void;
kbId: string;
}) => {
const [selecting, setSelecting] = useState(false);
const { toast } = useToast();
const { File, onOpen } = useSelectFile({ fileType: '.csv', multiple: false });
const [fileData, setFileData] = useState<{ q: string; a: string }[]>([]);
const [fileName, setFileName] = useState('');
const [successData, setSuccessData] = useState(0);
const { openConfirm, ConfirmChild } = useConfirm({
content: '确认导入该数据集?'
});
const onSelectFile = useCallback(
async (e: File[]) => {
const file = e[0];
setSelecting(true);
setFileName(file.name);
try {
const { header, data } = await readCsvContent(file);
if (header[0] !== 'question' || header[1] !== 'answer') {
throw new Error('csv 文件格式有误');
}
setFileData(
data.map((item) => ({
q: item[0] || '',
a: item[1] || ''
}))
);
} catch (error: any) {
toast({
title: getErrText(error, 'csv 文件格式有误'),
status: 'error'
});
}
setSelecting(false);
},
[setSelecting, toast]
);
const { mutate, isLoading: uploading } = useMutation({
mutationFn: async () => {
if (!fileData || fileData.length === 0) return;
let success = 0;
// subsection import
const step = 100;
for (let i = 0; i < fileData.length; i += step) {
const { insertLen } = await postKbDataFromList({
kbId,
data: fileData.slice(i, i + step).map((item) => ({
...item,
source: fileName
})),
mode: TrainingModeEnum.index
});
success += insertLen || 0;
setSuccessData((state) => state + step);
}
toast({
title: `导入数据成功,最终导入: ${success} 条数据。需要一段时间训练`,
status: 'success',
duration: 4000
});
onClose();
onSuccess();
},
onError(err) {
toast({
title: getErrText(err, '导入文件失败'),
status: 'error'
});
}
});
const { data: intro } = useMarkdown({ url: '/csvSelect.md' });
return (
<Modal isOpen={true} onClose={onClose} isCentered>
<ModalOverlay />
<ModalContent maxW={'90vw'} position={'relative'} m={0} h={'90vh'}>
<ModalHeader>csv </ModalHeader>
<ModalCloseButton />
<ModalBody h={'100%'} display={['block', 'flex']} fontSize={'sm'} overflowY={'auto'}>
<Box flex={'2 0 0'} w={['100%', 0]} mr={[0, 4]} mb={[4, 0]}>
<Markdown source={intro} />
<Box
my={3}
cursor={'pointer'}
textDecoration={'underline'}
color={'myBlue.600'}
onClick={() =>
fileDownload({
text: csvTemplate,
type: 'text/csv',
filename: 'template.csv'
})
}
>
csv模板
</Box>
<Box>
<Button isLoading={selecting} isDisabled={uploading} onClick={onOpen}>
csv
</Button>
<Box mt={4}>
{fileName} {fileData.length} 100
</Box>
</Box>
</Box>
<Box flex={'3 0 0'} h={'100%'} overflow={'auto'} p={2} backgroundColor={'blackAlpha.50'}>
{fileData.slice(0, 100).map((item, index) => (
<Box key={index}>
<Box>
Q{index + 1}. {item.q}
</Box>
<Box>
A{index + 1}. {item.a}
</Box>
</Box>
))}
</Box>
</ModalBody>
<Flex px={6} pt={2} pb={4}>
<Box flex={1}></Box>
<Button variant={'base'} isLoading={uploading} mr={3} onClick={onClose}>
</Button>
<Button isDisabled={fileData.length === 0 || uploading} onClick={openConfirm(mutate)}>
{uploading ? (
<Box>{Math.round((successData / fileData.length) * 100)}%</Box>
) : (
'确认导入'
)}
</Button>
</Flex>
</ModalContent>
<ConfirmChild />
<File onSelect={onSelectFile} />
</Modal>
);
};
export default SelectJsonModal;

View File

@@ -0,0 +1,349 @@
import React, { useState, useCallback } from 'react';
import {
Box,
Flex,
Button,
Modal,
ModalOverlay,
ModalContent,
ModalHeader,
ModalCloseButton,
ModalBody,
Input,
Textarea
} from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast';
import { useSelectFile } from '@/hooks/useSelectFile';
import { useConfirm } from '@/hooks/useConfirm';
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/file';
import { useMutation } from '@tanstack/react-query';
import { postKbDataFromList } from '@/api/plugins/kb';
import Radio from '@/components/Radio';
import { splitText_token } from '@/utils/file';
import { TrainingModeEnum } from '@/constants/plugin';
import { getErrText } from '@/utils/tools';
import { formatPrice } from '@/utils/user';
import MySlider from '@/components/Slider';
import { qaModelList, vectorModelList } from '@/store/static';
const fileExtension = '.txt,.doc,.docx,.pdf,.md';
const SelectFileModal = ({
onClose,
onSuccess,
kbId
}: {
onClose: () => void;
onSuccess: () => void;
kbId: string;
}) => {
const [modeMap, setModeMap] = useState({
[TrainingModeEnum.qa]: {
model: qaModelList[0].model,
maxLen: (qaModelList[0]?.maxToken || 16000) * 0.5,
price: qaModelList[0]?.price || 3
},
[TrainingModeEnum.index]: {
model: vectorModelList[0].model,
maxLen: 600,
price: vectorModelList[0]?.price || 0.2
}
});
const [btnLoading, setBtnLoading] = useState(false);
const { toast } = useToast();
const [prompt, setPrompt] = useState('');
const { File, onOpen } = useSelectFile({
fileType: fileExtension,
multiple: true
});
const [mode, setMode] = useState<`${TrainingModeEnum}`>(TrainingModeEnum.index);
const [files, setFiles] = useState<{ filename: string; text: string }[]>([
{ filename: '文本1', text: '' }
]);
const [splitRes, setSplitRes] = useState<{
price: number;
chunks: { filename: string; value: string }[];
successChunks: number;
}>({
price: 0,
successChunks: 0,
chunks: []
});
const { openConfirm, ConfirmChild } = useConfirm({
content: `确认导入该文件,需要一定时间进行拆解,该任务无法终止!如果余额不足,未完成的任务会被暂停。一共 ${
splitRes.chunks.length
} 组。${splitRes.price ? `大约 ${splitRes.price} 元。` : ''}`
});
const onSelectFile = useCallback(
async (files: File[]) => {
setBtnLoading(true);
try {
let promise = Promise.resolve();
files.forEach((file) => {
promise = promise.then(async () => {
const extension = file?.name?.split('.')?.pop()?.toLowerCase();
const text = await (async () => {
switch (extension) {
case 'txt':
case 'md':
return readTxtContent(file);
case 'pdf':
return readPdfContent(file);
case 'doc':
case 'docx':
return readDocContent(file);
}
return '';
})();
text && setFiles((state) => [{ filename: file.name, text }].concat(state));
return;
});
});
await promise;
} catch (error: any) {
console.log(error);
toast({
title: typeof error === 'string' ? error : '解析文件失败',
status: 'error'
});
}
setBtnLoading(false);
},
[toast]
);
console.log({ model: modeMap[mode].model });
const { mutate, isLoading: uploading } = useMutation({
mutationFn: async () => {
if (splitRes.chunks.length === 0) return;
// subsection import
let success = 0;
const step = 100;
for (let i = 0; i < splitRes.chunks.length; i += step) {
const { insertLen } = await postKbDataFromList({
kbId,
model: modeMap[mode].model,
data: splitRes.chunks
.slice(i, i + step)
.map((item) => ({ q: item.value, a: '', source: item.filename })),
prompt: `下面是"${prompt || '一段长文本'}"`,
mode
});
success += insertLen;
setSplitRes((state) => ({
...state,
successChunks: state.successChunks + step
}));
}
toast({
title: `去重后共导入 ${success} 条数据,需要一段拆解和训练.`,
status: 'success'
});
onClose();
onSuccess();
},
onError(err) {
toast({
title: getErrText(err, '导入文件失败'),
status: 'error'
});
}
});
const onclickImport = useCallback(async () => {
setBtnLoading(true);
try {
const splitRes = files
.map((item) =>
splitText_token({
text: item.text,
...modeMap[mode]
})
)
.map((item, i) => ({
...item,
filename: files[i].filename
}))
.filter((item) => item.tokens > 0);
let price = formatPrice(
splitRes.reduce((sum, item) => sum + item.tokens, 0) * modeMap[mode].price
);
if (mode === 'qa') {
price *= 1.2;
}
setSplitRes({
price,
chunks: splitRes
.map((item) =>
item.chunks.map((chunk) => ({
filename: item.filename,
value: chunk
}))
)
.flat(),
successChunks: 0
});
openConfirm(mutate)();
} catch (error) {
toast({
status: 'warning',
title: getErrText(error, '拆分文本异常')
});
}
setBtnLoading(false);
}, [files, mode, modeMap, mutate, openConfirm, toast]);
return (
<Modal isOpen={true} onClose={onClose} isCentered>
<ModalOverlay />
<ModalContent
display={'flex'}
maxW={'min(1000px, 90vw)'}
m={0}
position={'relative'}
h={'90vh'}
>
<ModalHeader></ModalHeader>
<ModalCloseButton />
<ModalBody
flex={1}
h={0}
display={'flex'}
flexDirection={'column'}
p={0}
alignItems={'center'}
justifyContent={'center'}
fontSize={'sm'}
>
<Box mt={2} px={5} maxW={['100%', '70%']} textAlign={'justify'} color={'blackAlpha.600'}>
{fileExtension} Gpt会自动对文本进行 QA
tokens{files.length}
</Box>
{/* 拆分模式 */}
<Flex w={'100%'} px={5} alignItems={'center'} mt={4}>
<Box flex={'0 0 70px'}>:</Box>
<Radio
ml={3}
list={[
{ label: '直接分段', value: 'index' },
{ label: 'QA拆分', value: 'qa' }
]}
value={mode}
onChange={(e) => setMode(e as 'index' | 'qa')}
/>
</Flex>
{/* 内容介绍 */}
<Flex w={'100%'} px={5} alignItems={'center'} mt={4}>
{mode === TrainingModeEnum.qa && (
<>
<Box flex={'0 0 70px'} mr={2}>
</Box>
<Input
placeholder="提示词,例如: Laf的介绍/关于gpt4的论文/一段长文本"
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
size={'sm'}
/>
</>
)}
{/* chunk size */}
{mode === TrainingModeEnum.index && (
<Flex mt={5}>
<Box w={['70px']} flexShrink={0}>
</Box>
<Box flex={1} ml={'10px'}>
<MySlider
markList={[
{ label: '300', value: 300 },
{ label: '1000', value: 1000 }
]}
width={['100%', '260px']}
min={300}
max={1000}
step={50}
value={modeMap[TrainingModeEnum.index].maxLen}
onChange={(val) => {
setModeMap((state) => ({
...state,
[TrainingModeEnum.index]: {
...modeMap[TrainingModeEnum.index],
maxLen: val
}
}));
}}
/>
</Box>
</Flex>
)}
</Flex>
{/* 文本内容 */}
<Box flex={'1 0 0'} px={5} h={0} w={'100%'} overflowY={'auto'} mt={4}>
{files.slice(0, 100).map((item, i) => (
<Box key={i} mb={5}>
<Box mb={1}>{item.filename}</Box>
<Textarea
placeholder="文件内容,空内容会自动忽略"
maxLength={-1}
rows={10}
fontSize={'xs'}
whiteSpace={'pre-wrap'}
value={item.text}
onChange={(e) => {
setFiles([
...files.slice(0, i),
{ ...item, text: e.target.value },
...files.slice(i + 1)
]);
}}
onBlur={(e) => {
if (files.length > 1 && e.target.value === '') {
setFiles((state) => [...state.slice(0, i), ...state.slice(i + 1)]);
}
}}
/>
</Box>
))}
</Box>
</ModalBody>
<Flex px={6} pt={2} pb={4}>
<Button isLoading={btnLoading} isDisabled={uploading} onClick={onOpen}>
</Button>
<Box flex={1}></Box>
<Button variant={'base'} isLoading={uploading} mr={3} onClick={onClose}>
</Button>
<Button
isDisabled={uploading || btnLoading || files[0]?.text === ''}
onClick={onclickImport}
>
{uploading ? (
<Box>{Math.round((splitRes.successChunks / splitRes.chunks.length) * 100)}%</Box>
) : (
'确认导入'
)}
</Button>
</Flex>
</ModalContent>
<ConfirmChild />
<File onSelect={onSelectFile} />
</Modal>
);
};
export default SelectFileModal;

View File

@@ -0,0 +1,288 @@
import React, { useEffect, useMemo, useState } from 'react';
import { Box, Textarea, Button, Flex, useTheme, Grid, Progress } from '@chakra-ui/react';
import { useKbStore } from '@/store/kb';
import type { KbTestItemType } from '@/types/plugin';
import { searchText, getKbDataItemById } from '@/api/plugins/kb';
import MyIcon from '@/components/Icon';
import { useRequest } from '@/hooks/useRequest';
import { useRouter } from 'next/router';
import { formatTimeToChatTime } from '@/utils/tools';
import InputDataModal, { type FormData } from './InputDataModal';
import { useGlobalStore } from '@/store/global';
import { getErrText } from '@/utils/tools';
import { useToast } from '@/hooks/useToast';
import { vectorModelList } from '@/store/static';
import { customAlphabet } from 'nanoid';
import MyTooltip from '@/components/MyTooltip';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const Test = () => {
const { kbId } = useRouter().query as { kbId: string };
const theme = useTheme();
const { toast } = useToast();
const { setLoading } = useGlobalStore();
const { kbTestList, pushKbTestItem, delKbTestItemById, updateKbItemById } = useKbStore();
const [inputText, setInputText] = useState('');
const [kbTestItem, setKbTestItem] = useState<KbTestItemType>();
const [editData, setEditData] = useState<FormData>();
const kbTestHistory = useMemo(
() => kbTestList.filter((item) => item.kbId === kbId),
[kbId, kbTestList]
);
const { mutate, isLoading } = useRequest({
mutationFn: () => searchText({ model: vectorModelList[0].model, kbId, text: inputText.trim() }),
onSuccess(res) {
const testItem = {
id: nanoid(),
kbId,
text: inputText.trim(),
time: new Date(),
results: res
};
pushKbTestItem(testItem);
setKbTestItem(testItem);
},
onError(err) {
toast({
title: getErrText(err),
status: 'error'
});
}
});
useEffect(() => {
setKbTestItem(undefined);
}, [kbId]);
return (
<Box h={'100%'} display={['block', 'flex']}>
<Box
h={['auto', '100%']}
display={['block', 'flex']}
flexDirection={'column'}
flex={1}
maxW={'500px'}
py={4}
borderRight={['none', theme.borders.base]}
>
<Box border={'2px solid'} borderColor={'myBlue.600'} p={3} mx={4} borderRadius={'md'}>
<Box fontSize={'sm'} fontWeight={'bold'}>
<MyIcon mr={2} name={'text'} w={'18px'} h={'18px'} color={'myBlue.700'} />
</Box>
<Textarea
rows={6}
resize={'none'}
variant={'unstyled'}
maxLength={1000}
placeholder="输入需要测试的文本"
value={inputText}
onChange={(e) => setInputText(e.target.value)}
/>
<Flex justifyContent={'flex-end'}>
<Button isDisabled={inputText === ''} isLoading={isLoading} onClick={mutate}>
</Button>
</Flex>
</Box>
<Box mt={5} flex={'1 0 0'} px={4} overflow={'overlay'} display={['none', 'block']}>
<Flex alignItems={'center'} color={'myGray.600'}>
<MyIcon mr={2} name={'history'} w={'16px'} h={'16px'} />
<Box fontSize={'2xl'}></Box>
</Flex>
<Box mt={2}>
<Flex py={2} fontWeight={'bold'} borderBottom={theme.borders.sm}>
<Box flex={1}></Box>
<Box w={'80px'}></Box>
<Box w={'14px'}></Box>
</Flex>
{kbTestHistory.map((item) => (
<Flex
key={item.id}
p={1}
alignItems={'center'}
borderBottom={theme.borders.base}
_hover={{
bg: '#f4f4f4',
'& .delete': {
display: 'block'
}
}}
cursor={'pointer'}
onClick={() => setKbTestItem(item)}
>
<Box flex={1} mr={2}>
{item.text}
</Box>
<Box w={'80px'}>{formatTimeToChatTime(item.time)}</Box>
<MyTooltip label={'删除该测试记录'}>
<Box w={'14px'} h={'14px'}>
<MyIcon
className="delete"
name={'delete'}
w={'14px'}
display={'none'}
_hover={{ color: 'red.600' }}
onClick={(e) => {
e.stopPropagation();
delKbTestItemById(item.id);
kbTestItem?.id === item.id && setKbTestItem(undefined);
}}
/>
</Box>
</MyTooltip>
</Flex>
))}
</Box>
</Box>
</Box>
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={1}>
{!kbTestItem?.results || kbTestItem.results.length === 0 ? (
<Flex
mt={[10, 0]}
h={'100%'}
flexDirection={'column'}
alignItems={'center'}
justifyContent={'center'}
>
<MyIcon name={'empty'} color={'transparent'} w={'54px'} />
<Box mt={3} color={'myGray.600'}>
</Box>
</Flex>
) : (
<>
<Flex alignItems={'flex-end'}>
<Box fontSize={'3xl'} color={'myGray.600'}>
</Box>
<MyTooltip
label={
'根据知识库内容与测试文本的相似度进行排序,你可以根据测试结果调整对应的文本。\n注意测试记录中的数据可能已经被修改过点击某条测试数据后将展示最新的数据。'
}
>
<QuestionOutlineIcon
ml={2}
color={'myGray.600'}
cursor={'pointer'}
fontSize={'lg'}
/>
</MyTooltip>
</Flex>
<Grid
mt={1}
gridTemplateColumns={[
'repeat(1,1fr)',
'repeat(1,1fr)',
'repeat(1,1fr)',
'repeat(2,1fr)'
]}
gridGap={4}
>
{kbTestItem?.results.map((item) => (
<Box
key={item.id}
pb={2}
borderRadius={'sm'}
border={theme.borders.base}
_notLast={{ mb: 2 }}
cursor={'pointer'}
title={'编辑'}
onClick={async () => {
try {
setLoading(true);
const data = await getKbDataItemById(item.id);
if (!data) {
throw new Error('该数据已被删除');
}
setEditData({
dataId: data.id,
q: data.q,
a: data.a
});
} catch (err) {
toast({
status: 'warning',
title: getErrText(err)
});
}
setLoading(false);
}}
>
<Flex p={3} alignItems={'center'} color={'myGray.500'}>
<MyIcon name={'kbTest'} w={'14px'} />
<Progress
mx={2}
flex={1}
value={item.score * 100}
size="sm"
borderRadius={'20px'}
colorScheme="gray"
/>
<Box>{item.score.toFixed(4)}</Box>
</Flex>
<Box
px={2}
fontSize={'xs'}
color={'myGray.600'}
maxH={'200px'}
overflow={'overlay'}
>
<Box>{item.q}</Box>
<Box>{item.a}</Box>
</Box>
</Box>
))}
</Grid>
</>
)}
</Box>
{editData && (
<InputDataModal
kbId={kbId}
defaultValues={editData}
onClose={() => setEditData(undefined)}
onSuccess={(data) => {
if (kbTestItem && editData.dataId) {
const newTestItem = {
...kbTestItem,
results: kbTestItem.results.map((item) =>
item.id === editData.dataId
? {
...item,
q: data.q,
a: data.a
}
: item
)
};
updateKbItemById(newTestItem);
setKbTestItem(newTestItem);
}
setEditData(undefined);
}}
onDelete={() => {
if (kbTestItem && editData.dataId) {
const newTestItem = {
...kbTestItem,
results: kbTestItem.results.filter((item) => item.id !== editData.dataId)
};
updateKbItemById(newTestItem);
setKbTestItem(newTestItem);
}
setEditData(undefined);
}}
/>
)}
</Box>
);
};
export default Test;

View File

@@ -0,0 +1,160 @@
import React, { useCallback, useMemo, useRef } from 'react';
import { useRouter } from 'next/router';
import { Box, Flex, IconButton, useTheme } from '@chakra-ui/react';
import { useToast } from '@/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useQuery } from '@tanstack/react-query';
import { useUserStore } from '@/store/user';
import { KbItemType } from '@/types/plugin';
import { useScreen } from '@/hooks/useScreen';
import { getErrText } from '@/utils/tools';
import { type ComponentRef } from './components/Info';
import Tabs from '@/components/Tabs';
import dynamic from 'next/dynamic';
import DataCard from './components/DataCard';
import MyIcon from '@/components/Icon';
import SideTabs from '@/components/SideTabs';
import PageContainer from '@/components/PageContainer';
import Avatar from '@/components/Avatar';
import Info from './components/Info';
const Test = dynamic(() => import('./components/Test'), {
ssr: false
});
enum TabEnum {
data = 'data',
test = 'test',
info = 'info'
}
const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` }) => {
const InfoRef = useRef<ComponentRef>(null);
const theme = useTheme();
const { toast } = useToast();
const router = useRouter();
const { isPc } = useScreen();
const { kbDetail, getKbDetail } = useUserStore();
const tabList = useMemo(
() => [
{ label: '数据集', id: TabEnum.data, icon: 'overviewLight' },
{ label: '搜索测试', id: TabEnum.test, icon: 'kbTest' },
{ label: '基本信息', id: TabEnum.info, icon: 'settingLight' }
],
[]
);
const setCurrentTab = useCallback(
(tab: `${TabEnum}`) => {
router.replace({
query: {
kbId,
currentTab: tab
}
});
},
[kbId, router]
);
const form = useForm<KbItemType>({
defaultValues: kbDetail
});
useQuery([kbId], () => getKbDetail(kbId), {
onSuccess(res) {
InfoRef.current?.initInput(res.tags);
form.reset(res);
},
onError(err: any) {
router.replace(`/kb/list`);
toast({
title: getErrText(err, '获取知识库异常'),
status: 'error'
});
}
});
return (
<PageContainer>
<Box display={['block', 'flex']} h={'100%'} pt={[4, 0]}>
{/* pc tab */}
<Box
display={['none', 'flex']}
flexDirection={'column'}
p={4}
w={'200px'}
borderRight={theme.borders.base}
>
<Flex mb={4} alignItems={'center'}>
<Avatar src={kbDetail.avatar} w={'34px'} borderRadius={'lg'} />
<Box ml={2} fontWeight={'bold'}>
{kbDetail.name}
</Box>
</Flex>
<SideTabs
flex={1}
mx={'auto'}
mt={2}
w={'100%'}
list={tabList}
activeId={currentTab}
onChange={(e: any) => {
setCurrentTab(e);
}}
/>
<Flex
alignItems={'center'}
cursor={'pointer'}
py={2}
px={3}
borderRadius={'md'}
_hover={{ bg: 'myGray.100' }}
onClick={() => router.replace('/kb/list')}
>
<IconButton
mr={3}
icon={<MyIcon name={'backFill'} w={'18px'} color={'myBlue.600'} />}
bg={'white'}
boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'}
h={'28px'}
size={'sm'}
borderRadius={'50%'}
aria-label={''}
/>
</Flex>
</Box>
<Box mb={3} display={['block', 'none']}>
<Tabs
m={'auto'}
w={'260px'}
size={isPc ? 'md' : 'sm'}
list={[
{ id: TabEnum.data, label: '数据管理' },
{ id: TabEnum.test, label: '搜索测试' },
{ id: TabEnum.info, label: '基本信息' }
]}
activeId={currentTab}
onChange={(e: any) => setCurrentTab(e)}
/>
</Box>
<Box flex={'1 0 0'} overflow={'overlay'} pb={[4, 0]}>
{currentTab === TabEnum.data && <DataCard kbId={kbId} />}
{currentTab === TabEnum.test && <Test />}
{currentTab === TabEnum.info && <Info ref={InfoRef} kbId={kbId} form={form} />}
</Box>
</Box>
</PageContainer>
);
};
export async function getServerSideProps(context: any) {
const currentTab = context?.query?.currentTab || TabEnum.data;
const kbId = context?.query?.kbId;
return {
props: { currentTab, kbId }
};
}
export default React.memo(Detail);