feat: 根据url获取网站文本
This commit is contained in:
archer
@@ -24,7 +24,7 @@ import { usePagination } from '@/hooks/usePagination';
|
||||
import {
|
||||
getModelDataList,
|
||||
delOneModelData,
|
||||
getModelSplitDataList,
|
||||
getModelSplitDataListLen,
|
||||
getExportDataList
|
||||
} from '@/api/model';
|
||||
import { DeleteIcon, RepeatIcon, EditIcon } from '@chakra-ui/icons';
|
||||
@@ -36,6 +36,7 @@ import type { FormData as InputDataType } from './InputDataModal';
|
||||
|
||||
const InputModel = dynamic(() => import('./InputDataModal'));
|
||||
const SelectFileModel = dynamic(() => import('./SelectFileModal'));
|
||||
const SelectUrlModel = dynamic(() => import('./SelectUrlModal'));
|
||||
const SelectJsonModel = dynamic(() => import('./SelectJsonModal'));
|
||||
|
||||
const ModelDataCard = ({ model }: { model: ModelSchema }) => {
|
||||
@@ -63,14 +64,19 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
|
||||
onOpen: onOpenSelectFileModal,
|
||||
onClose: onCloseSelectFileModal
|
||||
} = useDisclosure();
|
||||
const {
|
||||
isOpen: isOpenSelectUrlModal,
|
||||
onOpen: onOpenSelectUrlModal,
|
||||
onClose: onCloseSelectUrlModal
|
||||
} = useDisclosure();
|
||||
const {
|
||||
isOpen: isOpenSelectJsonModal,
|
||||
onOpen: onOpenSelectJsonModal,
|
||||
onClose: onCloseSelectJsonModal
|
||||
} = useDisclosure();
|
||||
|
||||
const { data: splitDataList, refetch } = useQuery(['getModelSplitDataList'], () =>
|
||||
getModelSplitDataList(model._id)
|
||||
const { data: splitDataLen, refetch } = useQuery(['getModelSplitDataList'], () =>
|
||||
getModelSplitDataListLen(model._id)
|
||||
);
|
||||
|
||||
const refetchData = useCallback(
|
||||
@@ -143,14 +149,13 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
|
||||
手动输入
|
||||
</MenuItem>
|
||||
<MenuItem onClick={onOpenSelectFileModal}>文件导入</MenuItem>
|
||||
<MenuItem onClick={onOpenSelectUrlModal}>网站地址导入</MenuItem>
|
||||
<MenuItem onClick={onOpenSelectJsonModal}>JSON导入</MenuItem>
|
||||
</MenuList>
|
||||
</Menu>
|
||||
</Flex>
|
||||
{splitDataList && splitDataList.length > 0 && (
|
||||
<Box fontSize={'xs'}>
|
||||
{splitDataList.map((item) => item.textList).flat().length}条数据正在拆分...
|
||||
</Box>
|
||||
{!!(splitDataLen && splitDataLen > 0) && (
|
||||
<Box fontSize={'xs'}>{splitDataLen}条数据正在拆分...</Box>
|
||||
)}
|
||||
<Box mt={4}>
|
||||
<TableContainer minH={'500px'}>
|
||||
@@ -236,6 +241,13 @@ const ModelDataCard = ({ model }: { model: ModelSchema }) => {
|
||||
onSuccess={refetchData}
|
||||
/>
|
||||
)}
|
||||
{isOpenSelectUrlModal && (
|
||||
<SelectUrlModel
|
||||
modelId={model._id}
|
||||
onClose={onCloseSelectUrlModal}
|
||||
onSuccess={refetchData}
|
||||
/>
|
||||
)}
|
||||
{isOpenSelectJsonModal && (
|
||||
<SelectJsonModel
|
||||
modelId={model._id}
|
||||
|
||||
@@ -19,7 +19,8 @@ import { encode } from 'gpt-token-utils';
|
||||
import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { readTxtContent, readPdfContent, readDocContent } from '@/utils/tools';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { postModelDataFileText } from '@/api/model';
|
||||
import { postModelDataSplitData } from '@/api/model';
|
||||
import { formatPrice } from '@/utils/user';
|
||||
|
||||
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
|
||||
|
||||
@@ -85,7 +86,7 @@ const SelectFileModal = ({
|
||||
const { mutate, isLoading } = useMutation({
|
||||
mutationFn: async () => {
|
||||
if (!fileText) return;
|
||||
await postModelDataFileText({
|
||||
await postModelDataSplitData({
|
||||
modelId,
|
||||
text: fileText,
|
||||
prompt: `下面是${prompt || '一段长文本'}`
|
||||
@@ -126,10 +127,11 @@ const SelectFileModal = ({
|
||||
</Button>
|
||||
<Box mt={2} maxW={['100%', '70%']}>
|
||||
支持 {fileExtension} 文件。模型会自动对文本进行 QA 拆分，需要较长训练时间，拆分需要消耗
|
||||
tokens，大约0.04元/1k tokens，请确保账号余额充足。
|
||||
tokens，账号余额不足时，未拆分的数据会被删除。
|
||||
</Box>
|
||||
<Box mt={2}>
|
||||
一共 {fileText.length} 个字，{encode(fileText).length} 个tokens
|
||||
一共 {encode(fileText).length} 个tokens，大约 {formatPrice(encode(fileText).length * 4)}
|
||||
元
|
||||
</Box>
|
||||
<Flex w={'100%'} alignItems={'center'} my={4}>
|
||||
<Box flex={'0 0 auto'} mr={2}>
|
||||
|
||||
168
src/pages/model/detail/components/SelectUrlModal.tsx
Normal file
168
src/pages/model/detail/components/SelectUrlModal.tsx
Normal file
@@ -0,0 +1,168 @@
|
||||
import React, { useState } from 'react';
|
||||
import {
|
||||
Box,
|
||||
Flex,
|
||||
Button,
|
||||
Modal,
|
||||
ModalOverlay,
|
||||
ModalContent,
|
||||
ModalHeader,
|
||||
ModalCloseButton,
|
||||
ModalBody,
|
||||
Input,
|
||||
Textarea
|
||||
} from '@chakra-ui/react';
|
||||
import { useToast } from '@/hooks/useToast';
|
||||
import { customAlphabet } from 'nanoid';
|
||||
import { encode } from 'gpt-token-utils';
|
||||
import { useConfirm } from '@/hooks/useConfirm';
|
||||
import { useMutation } from '@tanstack/react-query';
|
||||
import { postModelDataSplitData, getWebContent } from '@/api/model';
|
||||
import { formatPrice } from '@/utils/user';
|
||||
|
||||
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
|
||||
|
||||
const SelectUrlModal = ({
|
||||
onClose,
|
||||
onSuccess,
|
||||
modelId
|
||||
}: {
|
||||
onClose: () => void;
|
||||
onSuccess: () => void;
|
||||
modelId: string;
|
||||
}) => {
|
||||
const { toast } = useToast();
|
||||
const [webUrl, setWebUrl] = useState('');
|
||||
const [webText, setWebText] = useState('');
|
||||
const [prompt, setPrompt] = useState(''); // 提示词
|
||||
const { openConfirm, ConfirmChild } = useConfirm({
|
||||
content: '确认导入该文件，需要一定时间进行拆解，该任务无法终止！如果余额不足，任务讲被终止。'
|
||||
});
|
||||
|
||||
const { mutate: onclickImport, isLoading: isImporting } = useMutation({
|
||||
mutationFn: async () => {
|
||||
if (!webText) return;
|
||||
await postModelDataSplitData({
|
||||
modelId,
|
||||
text: webText,
|
||||
prompt: `下面是${prompt || '一段长文本'}`
|
||||
});
|
||||
toast({
|
||||
title: '导入数据成功,需要一段拆解和训练',
|
||||
status: 'success'
|
||||
});
|
||||
onClose();
|
||||
onSuccess();
|
||||
},
|
||||
onError(error) {
|
||||
console.log(error);
|
||||
toast({
|
||||
title: '导入数据失败',
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
const { mutate: onclickFetchingUrl, isLoading: isFetching } = useMutation({
|
||||
mutationFn: async () => {
|
||||
if (!webUrl) return;
|
||||
const res = await getWebContent(webUrl);
|
||||
const parser = new DOMParser();
|
||||
const htmlDoc = parser.parseFromString(res, 'text/html');
|
||||
const data = htmlDoc?.body?.innerText || '';
|
||||
|
||||
if (!data) {
|
||||
throw new Error('获取不到数据');
|
||||
}
|
||||
setWebText(data.replace(/\s+/g, ' '));
|
||||
},
|
||||
onError(error) {
|
||||
console.log(error);
|
||||
toast({
|
||||
status: 'error',
|
||||
title: '获取网站内容失败'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<Modal isOpen={true} onClose={onClose} isCentered>
|
||||
<ModalOverlay />
|
||||
<ModalContent maxW={'min(900px, 90vw)'} m={0} position={'relative'} h={'90vh'}>
|
||||
<ModalHeader>网站地址导入</ModalHeader>
|
||||
<ModalCloseButton />
|
||||
|
||||
<ModalBody
|
||||
display={'flex'}
|
||||
flexDirection={'column'}
|
||||
p={4}
|
||||
h={'100%'}
|
||||
alignItems={'center'}
|
||||
justifyContent={'center'}
|
||||
fontSize={'sm'}
|
||||
>
|
||||
<Box mt={2} maxW={['100%', '70%']}>
|
||||
根据网站地址，获取网站文本内容（请注意获取后的内容，不是每个网站内容都能获取到的）。模型会对文本进行
|
||||
QA 拆分，需要较长训练时间，拆分需要消耗 tokens，账号余额不足时，未拆分的数据会被删除。
|
||||
</Box>
|
||||
<Box mt={2}>
|
||||
一共 {encode(webText).length} 个tokens，大约 {formatPrice(encode(webText).length * 4)}元
|
||||
</Box>
|
||||
<Flex w={'100%'} alignItems={'center'} my={4}>
|
||||
<Box flex={'0 0 70px'}>网站地址</Box>
|
||||
<Input
|
||||
mx={2}
|
||||
placeholder="需要获取内容的地址。例如：https://fastgpt.ahapocket.cn"
|
||||
value={webUrl}
|
||||
onChange={(e) => setWebUrl(e.target.value)}
|
||||
size={'sm'}
|
||||
/>
|
||||
<Button isLoading={isFetching} onClick={() => onclickFetchingUrl()}>
|
||||
获取
|
||||
</Button>
|
||||
</Flex>
|
||||
<Flex w={'100%'} alignItems={'center'} my={4}>
|
||||
<Box flex={'0 0 70px'} mr={2}>
|
||||
下面是
|
||||
</Box>
|
||||
<Input
|
||||
placeholder="内容提示词。例如: Laf的介绍/关于gpt4的论文/一段长文本"
|
||||
value={prompt}
|
||||
onChange={(e) => setPrompt(e.target.value)}
|
||||
size={'sm'}
|
||||
/>
|
||||
</Flex>
|
||||
<Textarea
|
||||
flex={'1 0 0'}
|
||||
h={0}
|
||||
w={'100%'}
|
||||
placeholder="网站的内容"
|
||||
maxLength={-1}
|
||||
resize={'none'}
|
||||
fontSize={'xs'}
|
||||
whiteSpace={'pre-wrap'}
|
||||
value={webText}
|
||||
onChange={(e) => setWebText(e.target.value)}
|
||||
/>
|
||||
</ModalBody>
|
||||
|
||||
<Flex px={6} pt={2} pb={4}>
|
||||
<Box flex={1}></Box>
|
||||
<Button variant={'outline'} mr={3} onClick={onClose}>
|
||||
取消
|
||||
</Button>
|
||||
<Button
|
||||
isLoading={isImporting}
|
||||
isDisabled={webText === ''}
|
||||
onClick={openConfirm(onclickImport)}
|
||||
>
|
||||
确认导入
|
||||
</Button>
|
||||
</Flex>
|
||||
</ModalContent>
|
||||
<ConfirmChild />
|
||||
</Modal>
|
||||
);
|
||||
};
|
||||
|
||||
export default SelectUrlModal;
|
||||
Reference in New Issue
Block a user