V4.8.20 feature (#3686)
* Aiproxy (#3649) * model config * feat: model config ui * perf: rename variable * feat: custom request url * perf: model buffer * perf: init model * feat: json model config * auto login * fix: ts * update packages * package * fix: dockerfile * feat: usage filter & export & dashbord (#3538) * feat: usage filter & export & dashbord * adjust ui * fix tmb scroll * fix code & selecte all * merge * perf: usages list；perf: move components (#3654) * perf: usages list * team sub plan load * perf: usage dashboard code * perf: dashboard ui * perf: move components * add default model config (#3653) * 4.8.20 test (#3656) * provider * perf: model config * model perf (#3657) * fix: model * dataset quote * perf: model config * model tag * doubao model config * perf: config model * feat: model test * fix: POST 500 error on dingtalk bot (#3655) * feat: default model (#3662) * move model config * feat: default model * fix: false triggerd org selection (#3661) * export usage csv i18n (#3660) * export usage csv i18n * fix build * feat: markdown extension (#3663) * feat: markdown extension * media cros * rerank test * default price * perf: default model * fix: cannot custom provider * fix: default model select * update bg * perf: default model selector * fix: usage export * i18n * fix: rerank * update init extension * perf: ip limit check * doubao model order * web default modle * perf: tts selector * perf: tts error * qrcode package * reload buffer (#3665) * reload buffer * reload buffer * tts selector * fix: err tip (#3666) * fix: err tip * perf: training queue * doc * fix interactive edge (#3659) * fix interactive edge * fix * comment * add gemini model * fix: chat model select * perf: supplement assistant empty response (#3669) * perf: supplement assistant empty response * check array * perf: max_token count;feat: support resoner output;fix: member scroll (#3681) * perf: supplement assistant empty response * check array * perf: max_token count * feat: support resoner output * member scroll * update provider order * i18n * fix: stream response (#3682) * perf: supplement assistant empty response * check array * fix: stream response * fix: model config cannot set to null * fix: reasoning response (#3684) * perf: supplement assistant empty response * check array * fix: reasoning response * fix: reasoning response * doc (#3685) * perf: supplement assistant empty response * check array * doc * lock * animation * update doc * update compose * doc * doc --------- Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
This commit is contained in:
Archer
@@ -0,0 +1,317 @@
|
||||
import { useRouter } from 'next/router';
|
||||
import { SetStateAction, useState } from 'react';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { createContext, useContextSelector } from 'use-context-selector';
|
||||
import { ImportDataSourceEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { useMyStep } from '@fastgpt/web/hooks/useStep';
|
||||
import { Box, Button, Flex, IconButton } from '@chakra-ui/react';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { TabEnum } from '../NavBar';
|
||||
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
|
||||
import { UseFormReturn, useForm } from 'react-hook-form';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type';
|
||||
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
|
||||
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
|
||||
|
||||
type TrainingFiledType = {
|
||||
chunkOverlapRatio: number;
|
||||
maxChunkSize: number;
|
||||
minChunkSize: number;
|
||||
autoChunkSize: number;
|
||||
chunkSize: number;
|
||||
showChunkInput: boolean;
|
||||
showPromptInput: boolean;
|
||||
charsPointsPrice: number;
|
||||
priceTip: string;
|
||||
uploadRate: number;
|
||||
chunkSizeField?: ChunkSizeFieldType;
|
||||
};
|
||||
type DatasetImportContextType = {
|
||||
importSource: ImportDataSourceEnum;
|
||||
parentId: string | undefined;
|
||||
activeStep: number;
|
||||
goToNext: () => void;
|
||||
|
||||
processParamsForm: UseFormReturn<ImportFormType, any>;
|
||||
sources: ImportSourceItemType[];
|
||||
setSources: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
|
||||
} & TrainingFiledType;
|
||||
|
||||
type ChunkSizeFieldType = 'embeddingChunkSize' | 'qaChunkSize';
|
||||
export type ImportFormType = {
|
||||
mode: TrainingModeEnum;
|
||||
way: ImportProcessWayEnum;
|
||||
embeddingChunkSize: number;
|
||||
qaChunkSize: number;
|
||||
customSplitChar: string;
|
||||
qaPrompt: string;
|
||||
webSelector: string;
|
||||
};
|
||||
|
||||
export const DatasetImportContext = createContext<DatasetImportContextType>({
|
||||
importSource: ImportDataSourceEnum.fileLocal,
|
||||
goToNext: function (): void {
|
||||
throw new Error('Function not implemented.');
|
||||
},
|
||||
activeStep: 0,
|
||||
parentId: undefined,
|
||||
|
||||
maxChunkSize: 0,
|
||||
minChunkSize: 0,
|
||||
showChunkInput: false,
|
||||
showPromptInput: false,
|
||||
sources: [],
|
||||
setSources: function (value: SetStateAction<ImportSourceItemType[]>): void {
|
||||
throw new Error('Function not implemented.');
|
||||
},
|
||||
chunkSize: 0,
|
||||
chunkOverlapRatio: 0,
|
||||
uploadRate: 0,
|
||||
//@ts-ignore
|
||||
processParamsForm: undefined,
|
||||
autoChunkSize: 0,
|
||||
charsPointsPrice: 0,
|
||||
priceTip: ''
|
||||
});
|
||||
|
||||
const DatasetImportContextProvider = ({ children }: { children: React.ReactNode }) => {
|
||||
const { t } = useTranslation();
|
||||
const router = useRouter();
|
||||
const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as {
|
||||
source: ImportDataSourceEnum;
|
||||
parentId?: string;
|
||||
};
|
||||
|
||||
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
|
||||
|
||||
// step
|
||||
const modeSteps: Record<ImportDataSourceEnum, { title: string }[]> = {
|
||||
[ImportDataSourceEnum.reTraining]: [
|
||||
{ title: t('dataset:core.dataset.import.Adjust parameters') },
|
||||
{ title: t('common:core.dataset.import.Upload data') }
|
||||
],
|
||||
[ImportDataSourceEnum.fileLocal]: [
|
||||
{
|
||||
title: t('common:core.dataset.import.Select file')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Data Preprocessing')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Upload data')
|
||||
}
|
||||
],
|
||||
[ImportDataSourceEnum.fileLink]: [
|
||||
{
|
||||
title: t('common:core.dataset.import.Select file')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Data Preprocessing')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Upload data')
|
||||
}
|
||||
],
|
||||
[ImportDataSourceEnum.fileCustom]: [
|
||||
{
|
||||
title: t('common:core.dataset.import.Select file')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Data Preprocessing')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Upload data')
|
||||
}
|
||||
],
|
||||
[ImportDataSourceEnum.csvTable]: [
|
||||
{
|
||||
title: t('common:core.dataset.import.Select file')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Data Preprocessing')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Upload data')
|
||||
}
|
||||
],
|
||||
[ImportDataSourceEnum.externalFile]: [
|
||||
{
|
||||
title: t('common:core.dataset.import.Select file')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Data Preprocessing')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Upload data')
|
||||
}
|
||||
],
|
||||
[ImportDataSourceEnum.apiDataset]: [
|
||||
{
|
||||
title: t('common:core.dataset.import.Select file')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Data Preprocessing')
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Upload data')
|
||||
}
|
||||
]
|
||||
};
|
||||
const steps = modeSteps[source];
|
||||
const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({
|
||||
defaultStep: 0,
|
||||
steps
|
||||
});
|
||||
|
||||
const vectorModel = datasetDetail.vectorModel;
|
||||
const agentModel = datasetDetail.agentModel;
|
||||
|
||||
const processParamsForm = useForm<ImportFormType>({
|
||||
defaultValues: {
|
||||
mode: TrainingModeEnum.chunk,
|
||||
way: ImportProcessWayEnum.auto,
|
||||
embeddingChunkSize: vectorModel?.defaultToken || 512,
|
||||
qaChunkSize: Math.min(agentModel.maxResponse * 1, agentModel.maxContext * 0.7),
|
||||
customSplitChar: '',
|
||||
qaPrompt: Prompt_AgentQA.description,
|
||||
webSelector: ''
|
||||
}
|
||||
});
|
||||
|
||||
const [sources, setSources] = useState<ImportSourceItemType[]>([]);
|
||||
|
||||
// watch form
|
||||
const mode = processParamsForm.watch('mode');
|
||||
const way = processParamsForm.watch('way');
|
||||
const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize');
|
||||
const qaChunkSize = processParamsForm.watch('qaChunkSize');
|
||||
const customSplitChar = processParamsForm.watch('customSplitChar');
|
||||
|
||||
const modeStaticParams: Record<TrainingModeEnum, TrainingFiledType> = {
|
||||
[TrainingModeEnum.auto]: {
|
||||
chunkOverlapRatio: 0.2,
|
||||
maxChunkSize: 2048,
|
||||
minChunkSize: 100,
|
||||
autoChunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024,
|
||||
chunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024,
|
||||
showChunkInput: false,
|
||||
showPromptInput: false,
|
||||
charsPointsPrice: agentModel.charsPointsPrice || 0,
|
||||
priceTip: t('dataset:import.Auto mode Estimated Price Tips', {
|
||||
price: agentModel.charsPointsPrice
|
||||
}),
|
||||
uploadRate: 100
|
||||
},
|
||||
[TrainingModeEnum.chunk]: {
|
||||
chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType,
|
||||
chunkOverlapRatio: 0.2,
|
||||
maxChunkSize: vectorModel?.maxToken || 512,
|
||||
minChunkSize: 100,
|
||||
autoChunkSize: vectorModel?.defaultToken || 512,
|
||||
chunkSize: embeddingChunkSize,
|
||||
showChunkInput: true,
|
||||
showPromptInput: false,
|
||||
charsPointsPrice: vectorModel.charsPointsPrice || 0,
|
||||
priceTip: t('dataset:import.Embedding Estimated Price Tips', {
|
||||
price: vectorModel.charsPointsPrice
|
||||
}),
|
||||
uploadRate: 150
|
||||
},
|
||||
[TrainingModeEnum.qa]: {
|
||||
chunkSizeField: 'qaChunkSize' as ChunkSizeFieldType,
|
||||
chunkOverlapRatio: 0,
|
||||
maxChunkSize: Math.min(agentModel.maxResponse * 4, agentModel.maxContext * 0.7),
|
||||
minChunkSize: 4000,
|
||||
autoChunkSize: Math.min(agentModel.maxResponse * 1, agentModel.maxContext * 0.7),
|
||||
chunkSize: qaChunkSize,
|
||||
showChunkInput: true,
|
||||
showPromptInput: true,
|
||||
charsPointsPrice: agentModel.charsPointsPrice || 0,
|
||||
priceTip: t('dataset:import.Auto mode Estimated Price Tips', {
|
||||
price: agentModel.charsPointsPrice
|
||||
}),
|
||||
uploadRate: 30
|
||||
}
|
||||
};
|
||||
const selectModelStaticParam = modeStaticParams[mode];
|
||||
|
||||
const wayStaticPrams = {
|
||||
[ImportProcessWayEnum.auto]: {
|
||||
chunkSize: selectModelStaticParam.autoChunkSize,
|
||||
customSplitChar: ''
|
||||
},
|
||||
[ImportProcessWayEnum.custom]: {
|
||||
chunkSize: modeStaticParams[mode].chunkSize,
|
||||
customSplitChar
|
||||
}
|
||||
};
|
||||
const chunkSize = wayStaticPrams[way].chunkSize;
|
||||
|
||||
const contextValue = {
|
||||
importSource: source,
|
||||
parentId,
|
||||
activeStep,
|
||||
goToNext,
|
||||
|
||||
processParamsForm,
|
||||
...selectModelStaticParam,
|
||||
sources,
|
||||
setSources,
|
||||
chunkSize
|
||||
};
|
||||
|
||||
return (
|
||||
<DatasetImportContext.Provider value={contextValue}>
|
||||
<Flex>
|
||||
{activeStep === 0 ? (
|
||||
<Flex alignItems={'center'}>
|
||||
<IconButton
|
||||
icon={<MyIcon name={'common/backFill'} w={'14px'} />}
|
||||
aria-label={''}
|
||||
size={'smSquare'}
|
||||
borderRadius={'50%'}
|
||||
variant={'whiteBase'}
|
||||
mr={2}
|
||||
onClick={() =>
|
||||
router.replace({
|
||||
query: {
|
||||
...router.query,
|
||||
currentTab: TabEnum.collectionCard
|
||||
}
|
||||
})
|
||||
}
|
||||
/>
|
||||
{t('common:common.Exit')}
|
||||
</Flex>
|
||||
) : (
|
||||
<Button
|
||||
variant={'whiteBase'}
|
||||
leftIcon={<MyIcon name={'common/backFill'} w={'14px'} />}
|
||||
onClick={goToPrevious}
|
||||
>
|
||||
{t('common:common.Last Step')}
|
||||
</Button>
|
||||
)}
|
||||
<Box flex={1} />
|
||||
</Flex>
|
||||
{/* step */}
|
||||
<Box
|
||||
mt={4}
|
||||
mb={5}
|
||||
px={3}
|
||||
py={[2, 4]}
|
||||
bg={'myGray.50'}
|
||||
borderWidth={'1px'}
|
||||
borderColor={'borderColor.low'}
|
||||
borderRadius={'md'}
|
||||
>
|
||||
<Box maxW={['100%', '900px']} mx={'auto'}>
|
||||
<MyStep />
|
||||
</Box>
|
||||
</Box>
|
||||
{children}
|
||||
</DatasetImportContext.Provider>
|
||||
);
|
||||
};
|
||||
|
||||
export default DatasetImportContextProvider;
|
||||
@@ -0,0 +1,319 @@
|
||||
import React, { useCallback, useMemo, useRef } from 'react';
|
||||
import {
|
||||
Box,
|
||||
Flex,
|
||||
Input,
|
||||
Button,
|
||||
ModalBody,
|
||||
ModalFooter,
|
||||
Textarea,
|
||||
useDisclosure
|
||||
} from '@chakra-ui/react';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
|
||||
import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
|
||||
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
|
||||
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import MyModal from '@fastgpt/web/components/common/MyModal';
|
||||
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
|
||||
import Preview from '../components/Preview';
|
||||
import MyTag from '@fastgpt/web/components/common/Tag/index';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
import { useToast } from '@fastgpt/web/hooks/useToast';
|
||||
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
|
||||
import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
|
||||
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
|
||||
|
||||
function DataProcess({ showPreviewChunks = true }: { showPreviewChunks: boolean }) {
|
||||
const { t } = useTranslation();
|
||||
const { feConfigs } = useSystemStore();
|
||||
|
||||
const {
|
||||
goToNext,
|
||||
processParamsForm,
|
||||
chunkSizeField,
|
||||
minChunkSize,
|
||||
showChunkInput,
|
||||
showPromptInput,
|
||||
maxChunkSize,
|
||||
priceTip,
|
||||
chunkSize
|
||||
} = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const { getValues, setValue, register, watch } = processParamsForm;
|
||||
const { toast } = useToast();
|
||||
const mode = watch('mode');
|
||||
const way = watch('way');
|
||||
|
||||
const {
|
||||
isOpen: isOpenCustomPrompt,
|
||||
onOpen: onOpenCustomPrompt,
|
||||
onClose: onCloseCustomPrompt
|
||||
} = useDisclosure();
|
||||
|
||||
const trainingModeList = useMemo(() => {
|
||||
const list = Object.entries(TrainingTypeMap);
|
||||
return list;
|
||||
}, []);
|
||||
|
||||
const onSelectTrainWay = useCallback(
|
||||
(e: TrainingModeEnum) => {
|
||||
if (!feConfigs?.isPlus && !TrainingTypeMap[e]?.openSource) {
|
||||
return toast({
|
||||
status: 'warning',
|
||||
title: t('common:common.system.Commercial version function')
|
||||
});
|
||||
}
|
||||
setValue('mode', e);
|
||||
},
|
||||
[feConfigs?.isPlus, setValue, t, toast]
|
||||
);
|
||||
|
||||
return (
|
||||
<Box h={'100%'} display={['block', 'flex']} fontSize={'sm'}>
|
||||
<Box
|
||||
flex={'1 0 0'}
|
||||
minW={['auto', '500px']}
|
||||
maxW={'600px'}
|
||||
h={['auto', '100%']}
|
||||
overflow={'auto'}
|
||||
pr={[0, 3]}
|
||||
>
|
||||
<Flex alignItems={'center'}>
|
||||
<MyIcon name={'common/settingLight'} w={'20px'} />
|
||||
<Box fontSize={'md'}>{t('dataset:data_process_setting')}</Box>
|
||||
</Flex>
|
||||
|
||||
<Box display={['block', 'flex']} mt={4} alignItems={'center'}>
|
||||
<FormLabel flex={'0 0 100px'}>{t('dataset:training_mode')}</FormLabel>
|
||||
<LeftRadio
|
||||
list={trainingModeList.map(([key, value]) => ({
|
||||
title: t(value.label as any),
|
||||
value: key,
|
||||
tooltip: t(value.tooltip as any)
|
||||
}))}
|
||||
px={3}
|
||||
py={2}
|
||||
value={mode}
|
||||
onChange={onSelectTrainWay}
|
||||
defaultBg="white"
|
||||
activeBg="white"
|
||||
display={'flex'}
|
||||
flexWrap={'wrap'}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
<Box display={['block', 'flex']} mt={5}>
|
||||
<FormLabel flex={'0 0 100px'}>{t('dataset:data_process_params')}</FormLabel>
|
||||
<LeftRadio
|
||||
list={[
|
||||
{
|
||||
title: t('common:core.dataset.import.Auto process'),
|
||||
desc: t('common:core.dataset.import.Auto process desc'),
|
||||
value: ImportProcessWayEnum.auto
|
||||
},
|
||||
{
|
||||
title: t('dataset:custom_data_process_params'),
|
||||
desc: t('dataset:custom_data_process_params_desc'),
|
||||
value: ImportProcessWayEnum.custom,
|
||||
children: way === ImportProcessWayEnum.custom && (
|
||||
<Box mt={5}>
|
||||
{showChunkInput && chunkSizeField && (
|
||||
<Box>
|
||||
<Flex alignItems={'center'}>
|
||||
<Box>{t('dataset:ideal_chunk_length')}</Box>
|
||||
<QuestionTip label={t('dataset:ideal_chunk_length_tips')} />
|
||||
</Flex>
|
||||
<Box
|
||||
mt={1}
|
||||
css={{
|
||||
'& > span': {
|
||||
display: 'block'
|
||||
}
|
||||
}}
|
||||
>
|
||||
<MyTooltip
|
||||
label={t('common:core.dataset.import.Chunk Range', {
|
||||
min: minChunkSize,
|
||||
max: maxChunkSize
|
||||
})}
|
||||
>
|
||||
<MyNumberInput
|
||||
name={chunkSizeField}
|
||||
min={minChunkSize}
|
||||
max={maxChunkSize}
|
||||
size={'sm'}
|
||||
step={100}
|
||||
value={chunkSize}
|
||||
onChange={(e) => {
|
||||
if (e === undefined) return;
|
||||
setValue(chunkSizeField, +e);
|
||||
}}
|
||||
/>
|
||||
</MyTooltip>
|
||||
</Box>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
<Box mt={3}>
|
||||
<Box>
|
||||
{t('common:core.dataset.import.Custom split char')}
|
||||
<QuestionTip
|
||||
label={t('common:core.dataset.import.Custom split char Tips')}
|
||||
/>
|
||||
</Box>
|
||||
<Box mt={1}>
|
||||
<Input
|
||||
size={'sm'}
|
||||
bg={'myGray.50'}
|
||||
defaultValue={''}
|
||||
placeholder="\n;======;==SPLIT=="
|
||||
{...register('customSplitChar')}
|
||||
/>
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
{showPromptInput && (
|
||||
<Box mt={3}>
|
||||
<Box>{t('common:core.dataset.collection.QA Prompt')}</Box>
|
||||
<Box
|
||||
position={'relative'}
|
||||
py={2}
|
||||
px={3}
|
||||
bg={'myGray.50'}
|
||||
fontSize={'xs'}
|
||||
whiteSpace={'pre-wrap'}
|
||||
border={'1px'}
|
||||
borderColor={'borderColor.base'}
|
||||
borderRadius={'md'}
|
||||
maxH={'140px'}
|
||||
overflow={'auto'}
|
||||
_hover={{
|
||||
'& .mask': {
|
||||
display: 'block'
|
||||
}
|
||||
}}
|
||||
>
|
||||
{getValues('qaPrompt')}
|
||||
|
||||
<Box
|
||||
display={'none'}
|
||||
className="mask"
|
||||
position={'absolute'}
|
||||
top={0}
|
||||
right={0}
|
||||
bottom={0}
|
||||
left={0}
|
||||
background={
|
||||
'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
|
||||
}
|
||||
>
|
||||
<Button
|
||||
size="xs"
|
||||
variant={'whiteBase'}
|
||||
leftIcon={<MyIcon name={'edit'} w={'13px'} />}
|
||||
color={'black'}
|
||||
position={'absolute'}
|
||||
right={2}
|
||||
bottom={2}
|
||||
onClick={onOpenCustomPrompt}
|
||||
>
|
||||
{t('common:core.dataset.import.Custom prompt')}
|
||||
</Button>
|
||||
</Box>
|
||||
</Box>
|
||||
</Box>
|
||||
)}
|
||||
</Box>
|
||||
)
|
||||
}
|
||||
]}
|
||||
px={3}
|
||||
py={3}
|
||||
defaultBg="white"
|
||||
activeBg="white"
|
||||
value={way}
|
||||
w={'100%'}
|
||||
onChange={(e) => {
|
||||
setValue('way', e);
|
||||
}}
|
||||
></LeftRadio>
|
||||
</Box>
|
||||
|
||||
{feConfigs?.show_pay && (
|
||||
<Box mt={5} pl={[0, '100px']} gap={3}>
|
||||
<MyTag colorSchema={'gray'} py={1.5} borderRadius={'md'} px={3} whiteSpace={'wrap'}>
|
||||
{priceTip}
|
||||
</MyTag>
|
||||
</Box>
|
||||
)}
|
||||
|
||||
<Flex mt={5} gap={3} justifyContent={'flex-end'}>
|
||||
<Button
|
||||
onClick={() => {
|
||||
goToNext();
|
||||
}}
|
||||
>
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Box>
|
||||
<Box flex={'1 0 0'} w={['auto', '0']} h={['auto', '100%']} pl={[0, 3]}>
|
||||
<Preview showPreviewChunks={showPreviewChunks} />
|
||||
</Box>
|
||||
|
||||
{isOpenCustomPrompt && (
|
||||
<PromptTextarea
|
||||
defaultValue={getValues('qaPrompt')}
|
||||
onChange={(e) => {
|
||||
setValue('qaPrompt', e);
|
||||
}}
|
||||
onClose={onCloseCustomPrompt}
|
||||
/>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
export default React.memo(DataProcess);
|
||||
|
||||
const PromptTextarea = ({
|
||||
defaultValue,
|
||||
onChange,
|
||||
onClose
|
||||
}: {
|
||||
defaultValue: string;
|
||||
onChange: (e: string) => void;
|
||||
onClose: () => void;
|
||||
}) => {
|
||||
const ref = useRef<HTMLTextAreaElement>(null);
|
||||
const { t } = useTranslation();
|
||||
|
||||
return (
|
||||
<MyModal
|
||||
isOpen
|
||||
title={t('common:core.dataset.import.Custom prompt')}
|
||||
iconSrc="modal/edit"
|
||||
w={'600px'}
|
||||
onClose={onClose}
|
||||
>
|
||||
<ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
|
||||
<Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
|
||||
<Box>{Prompt_AgentQA.fixedText}</Box>
|
||||
</ModalBody>
|
||||
<ModalFooter>
|
||||
<Button
|
||||
onClick={() => {
|
||||
const val = ref.current?.value || Prompt_AgentQA.description;
|
||||
onChange(val);
|
||||
onClose();
|
||||
}}
|
||||
>
|
||||
{t('common:common.Confirm')}
|
||||
</Button>
|
||||
</ModalFooter>
|
||||
</MyModal>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,24 @@
|
||||
import React from 'react';
|
||||
import Preview from '../components/Preview';
|
||||
import { Box, Button, Flex } from '@chakra-ui/react';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
const PreviewData = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => {
|
||||
const { t } = useTranslation();
|
||||
const goToNext = useContextSelector(DatasetImportContext, (v) => v.goToNext);
|
||||
|
||||
return (
|
||||
<Flex flexDirection={'column'} h={'100%'}>
|
||||
<Box flex={'1 0 0 '}>
|
||||
<Preview showPreviewChunks={showPreviewChunks} />
|
||||
</Box>
|
||||
<Flex mt={2} justifyContent={'flex-end'}>
|
||||
<Button onClick={goToNext}>{t('common:common.Next Step')}</Button>
|
||||
</Flex>
|
||||
</Flex>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(PreviewData);
|
||||
@@ -0,0 +1,285 @@
|
||||
import React, { useMemo, useRef } from 'react';
|
||||
import { QuestionOutlineIcon } from '@chakra-ui/icons';
|
||||
import {
|
||||
Box,
|
||||
TableContainer,
|
||||
Table,
|
||||
Thead,
|
||||
Tr,
|
||||
Th,
|
||||
Td,
|
||||
Tbody,
|
||||
Flex,
|
||||
Button,
|
||||
IconButton,
|
||||
Tooltip
|
||||
} from '@chakra-ui/react';
|
||||
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
import { useToast } from '@fastgpt/web/hooks/useToast';
|
||||
import { useRouter } from 'next/router';
|
||||
import { TabEnum } from '../../../../../pages/dataset/detail/index';
|
||||
import {
|
||||
postCreateDatasetApiDatasetCollection,
|
||||
postCreateDatasetCsvTableCollection,
|
||||
postCreateDatasetExternalFileCollection,
|
||||
postCreateDatasetFileCollection,
|
||||
postCreateDatasetLinkCollection,
|
||||
postCreateDatasetTextCollection,
|
||||
postReTrainingDatasetFileCollection
|
||||
} from '@/web/core/dataset/api';
|
||||
import MyTag from '@fastgpt/web/components/common/Tag/index';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
|
||||
import { DatasetImportContext, type ImportFormType } from '../Context';
|
||||
|
||||
const Upload = () => {
|
||||
const { t } = useTranslation();
|
||||
const { toast } = useToast();
|
||||
const router = useRouter();
|
||||
const { collectionId = '' } = router.query as {
|
||||
collectionId: string;
|
||||
};
|
||||
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
|
||||
const retrainNewCollectionId = useRef('');
|
||||
|
||||
const { importSource, parentId, sources, setSources, processParamsForm, chunkSize } =
|
||||
useContextSelector(DatasetImportContext, (v) => v);
|
||||
|
||||
const { handleSubmit } = processParamsForm;
|
||||
|
||||
const { totalFilesCount, waitingFilesCount, allFinished, hasCreatingFiles } = useMemo(() => {
|
||||
const totalFilesCount = sources.length;
|
||||
|
||||
const { waitingFilesCount, allFinished, hasCreatingFiles } = sources.reduce(
|
||||
(acc, file) => {
|
||||
if (file.createStatus === 'waiting') acc.waitingFilesCount++;
|
||||
if (file.createStatus === 'creating') acc.hasCreatingFiles = true;
|
||||
if (file.createStatus !== 'finish') acc.allFinished = false;
|
||||
return acc;
|
||||
},
|
||||
{ waitingFilesCount: 0, allFinished: true, hasCreatingFiles: false }
|
||||
);
|
||||
|
||||
return { totalFilesCount, waitingFilesCount, allFinished, hasCreatingFiles };
|
||||
}, [sources]);
|
||||
|
||||
const buttonText = useMemo(() => {
|
||||
if (waitingFilesCount === totalFilesCount) {
|
||||
return t('common:core.dataset.import.Start upload');
|
||||
} else if (allFinished) {
|
||||
return t('common:core.dataset.import.Upload complete');
|
||||
} else {
|
||||
return t('common:core.dataset.import.Continue upload');
|
||||
}
|
||||
}, [waitingFilesCount, totalFilesCount, allFinished, t]);
|
||||
|
||||
const { runAsync: startUpload, loading: isLoading } = useRequest2(
|
||||
async ({ mode, customSplitChar, qaPrompt, webSelector }: ImportFormType) => {
|
||||
if (sources.length === 0) return;
|
||||
const filterWaitingSources = sources.filter((item) => item.createStatus === 'waiting');
|
||||
|
||||
// Batch create collection and upload chunks
|
||||
for await (const item of filterWaitingSources) {
|
||||
setSources((state) =>
|
||||
state.map((source) =>
|
||||
source.id === item.id
|
||||
? {
|
||||
...source,
|
||||
createStatus: 'creating'
|
||||
}
|
||||
: source
|
||||
)
|
||||
);
|
||||
|
||||
// create collection
|
||||
const commonParams = {
|
||||
parentId,
|
||||
trainingType: mode,
|
||||
datasetId: datasetDetail._id,
|
||||
chunkSize,
|
||||
chunkSplitter: customSplitChar,
|
||||
qaPrompt,
|
||||
|
||||
name: item.sourceName
|
||||
};
|
||||
if (importSource === ImportDataSourceEnum.reTraining) {
|
||||
const res = await postReTrainingDatasetFileCollection({
|
||||
...commonParams,
|
||||
collectionId
|
||||
});
|
||||
retrainNewCollectionId.current = res.collectionId;
|
||||
} else if (importSource === ImportDataSourceEnum.fileLocal && item.dbFileId) {
|
||||
await postCreateDatasetFileCollection({
|
||||
...commonParams,
|
||||
fileId: item.dbFileId
|
||||
});
|
||||
} else if (importSource === ImportDataSourceEnum.fileLink && item.link) {
|
||||
await postCreateDatasetLinkCollection({
|
||||
...commonParams,
|
||||
link: item.link,
|
||||
metadata: {
|
||||
webPageSelector: webSelector
|
||||
}
|
||||
});
|
||||
} else if (importSource === ImportDataSourceEnum.fileCustom && item.rawText) {
|
||||
// manual collection
|
||||
await postCreateDatasetTextCollection({
|
||||
...commonParams,
|
||||
text: item.rawText
|
||||
});
|
||||
} else if (importSource === ImportDataSourceEnum.csvTable && item.dbFileId) {
|
||||
await postCreateDatasetCsvTableCollection({
|
||||
...commonParams,
|
||||
fileId: item.dbFileId
|
||||
});
|
||||
} else if (importSource === ImportDataSourceEnum.externalFile && item.externalFileUrl) {
|
||||
await postCreateDatasetExternalFileCollection({
|
||||
...commonParams,
|
||||
externalFileUrl: item.externalFileUrl,
|
||||
externalFileId: item.externalFileId,
|
||||
filename: item.sourceName
|
||||
});
|
||||
} else if (importSource === ImportDataSourceEnum.apiDataset && item.apiFileId) {
|
||||
await postCreateDatasetApiDatasetCollection({
|
||||
...commonParams,
|
||||
apiFileId: item.apiFileId
|
||||
});
|
||||
}
|
||||
|
||||
setSources((state) =>
|
||||
state.map((source) =>
|
||||
source.id === item.id
|
||||
? {
|
||||
...source,
|
||||
createStatus: 'finish'
|
||||
}
|
||||
: source
|
||||
)
|
||||
);
|
||||
}
|
||||
},
|
||||
{
|
||||
onSuccess() {
|
||||
if (!sources.some((file) => file.errorMsg !== undefined)) {
|
||||
toast({
|
||||
title:
|
||||
importSource === ImportDataSourceEnum.reTraining
|
||||
? t('dataset:retrain_task_submitted')
|
||||
: t('common:core.dataset.import.Import success'),
|
||||
status: 'success'
|
||||
});
|
||||
}
|
||||
|
||||
// Close import page
|
||||
router.replace({
|
||||
query: {
|
||||
datasetId: datasetDetail._id,
|
||||
currentTab: TabEnum.collectionCard
|
||||
}
|
||||
});
|
||||
},
|
||||
onError(error) {
|
||||
setSources((state) =>
|
||||
state.map((source) =>
|
||||
source.createStatus === 'creating'
|
||||
? {
|
||||
...source,
|
||||
createStatus: 'waiting',
|
||||
errorMsg: error.message || t('file:upload_failed')
|
||||
}
|
||||
: source
|
||||
)
|
||||
);
|
||||
},
|
||||
errorToast: t('file:upload_failed')
|
||||
}
|
||||
);
|
||||
|
||||
return (
|
||||
<Box h={'100%'} overflow={'auto'}>
|
||||
<TableContainer>
|
||||
<Table variant={'simple'} fontSize={'sm'} draggable={false}>
|
||||
<Thead draggable={false}>
|
||||
<Tr bg={'myGray.100'} mb={2}>
|
||||
<Th borderLeftRadius={'md'} overflow={'hidden'} borderBottom={'none'} py={4}>
|
||||
{t('common:core.dataset.import.Source name')}
|
||||
</Th>
|
||||
<Th borderBottom={'none'} py={4}>
|
||||
{t('common:core.dataset.import.Upload status')}
|
||||
</Th>
|
||||
<Th borderRightRadius={'md'} borderBottom={'none'} py={4}>
|
||||
{t('common:common.Action')}
|
||||
</Th>
|
||||
</Tr>
|
||||
</Thead>
|
||||
<Tbody>
|
||||
{sources.map((item) => (
|
||||
<Tr key={item.id}>
|
||||
<Td>
|
||||
<Flex alignItems={'center'}>
|
||||
<MyIcon name={item.icon as any} w={'16px'} mr={1} />
|
||||
<Box whiteSpace={'wrap'} maxW={'30vw'}>
|
||||
{item.sourceName}
|
||||
</Box>
|
||||
</Flex>
|
||||
</Td>
|
||||
<Td>
|
||||
<Box display={'inline-block'}>
|
||||
{item.errorMsg ? (
|
||||
<Tooltip label={item.errorMsg} fontSize="md">
|
||||
<Flex alignItems="center">
|
||||
<MyTag colorSchema={'red'}>{t('common:common.Error')}</MyTag>
|
||||
<QuestionOutlineIcon ml={2} color="red.500" w="14px" />
|
||||
</Flex>
|
||||
</Tooltip>
|
||||
) : (
|
||||
<>
|
||||
{item.createStatus === 'waiting' && (
|
||||
<MyTag colorSchema={'gray'}>{t('common:common.Waiting')}</MyTag>
|
||||
)}
|
||||
{item.createStatus === 'creating' && (
|
||||
<MyTag colorSchema={'blue'}>{t('common:common.Creating')}</MyTag>
|
||||
)}
|
||||
{item.createStatus === 'finish' && (
|
||||
<MyTag colorSchema={'green'}>{t('common:common.Finish')}</MyTag>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
</Td>
|
||||
<Td>
|
||||
{!hasCreatingFiles && item.createStatus !== 'finish' && (
|
||||
<IconButton
|
||||
variant={'grayDanger'}
|
||||
size={'sm'}
|
||||
icon={<MyIcon name={'delete'} w={'14px'} />}
|
||||
aria-label={'Delete file'}
|
||||
onClick={() => {
|
||||
setSources((prevFiles) => prevFiles.filter((file) => file.id !== item.id));
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
</Tbody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
|
||||
<Flex justifyContent={'flex-end'} mt={4}>
|
||||
<Button isLoading={isLoading} onClick={handleSubmit((data) => startUpload(data))}>
|
||||
{totalFilesCount > 0 &&
|
||||
`${t('common:core.dataset.import.Total files', {
|
||||
total: totalFilesCount
|
||||
})} | `}
|
||||
{buttonText}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default Upload;
|
||||
@@ -0,0 +1,336 @@
|
||||
import MyBox from '@fastgpt/web/components/common/MyBox';
|
||||
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
|
||||
import { useToast } from '@fastgpt/web/hooks/useToast';
|
||||
import { Box, FlexProps } from '@chakra-ui/react';
|
||||
import { formatFileSize } from '@fastgpt/global/common/file/tools';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import React, { DragEvent, useCallback, useMemo, useState } from 'react';
|
||||
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
||||
import { useRequest } from '@fastgpt/web/hooks/useRequest';
|
||||
import { getFileIcon } from '@fastgpt/global/common/file/icon';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import { uploadFile2DB } from '@/web/common/file/controller';
|
||||
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type';
|
||||
import { useI18n } from '@/web/context/I18n';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
|
||||
|
||||
export type SelectFileItemType = {
|
||||
fileId: string;
|
||||
folderPath: string;
|
||||
file: File;
|
||||
};
|
||||
|
||||
const FileSelector = ({
|
||||
fileType,
|
||||
selectFiles,
|
||||
setSelectFiles,
|
||||
onStartSelect,
|
||||
onFinishSelect,
|
||||
...props
|
||||
}: {
|
||||
fileType: string;
|
||||
selectFiles: ImportSourceItemType[];
|
||||
setSelectFiles: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
|
||||
onStartSelect: () => void;
|
||||
onFinishSelect: () => void;
|
||||
} & FlexProps) => {
|
||||
const { t } = useTranslation();
|
||||
const { fileT } = useI18n();
|
||||
|
||||
const { toast } = useToast();
|
||||
const { feConfigs } = useSystemStore();
|
||||
|
||||
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
|
||||
const maxCount = feConfigs?.uploadFileMaxAmount || 1000;
|
||||
const maxSize = (feConfigs?.uploadFileMaxSize || 1024) * 1024 * 1024;
|
||||
|
||||
const { File, onOpen } = useSelectFile({
|
||||
fileType,
|
||||
multiple: true,
|
||||
maxCount
|
||||
});
|
||||
const [isDragging, setIsDragging] = useState(false);
|
||||
const isMaxSelected = useMemo(
|
||||
() => selectFiles.length >= maxCount,
|
||||
[maxCount, selectFiles.length]
|
||||
);
|
||||
|
||||
const filterTypeReg = new RegExp(
|
||||
`(${fileType
|
||||
.split(',')
|
||||
.map((item) => item.trim())
|
||||
.join('|')})$`,
|
||||
'i'
|
||||
);
|
||||
|
||||
const { mutate: onSelectFile, isLoading } = useRequest({
|
||||
mutationFn: async (files: SelectFileItemType[]) => {
|
||||
{
|
||||
onStartSelect();
|
||||
setSelectFiles((state) => {
|
||||
const formatFiles = files.map<ImportSourceItemType>((selectFile) => {
|
||||
const { fileId, file } = selectFile;
|
||||
|
||||
return {
|
||||
id: fileId,
|
||||
createStatus: 'waiting',
|
||||
file,
|
||||
sourceName: file.name,
|
||||
sourceSize: formatFileSize(file.size),
|
||||
icon: getFileIcon(file.name),
|
||||
isUploading: true,
|
||||
uploadedFileRate: 0
|
||||
};
|
||||
});
|
||||
const results = formatFiles.concat(state).slice(0, maxCount);
|
||||
return results;
|
||||
});
|
||||
try {
|
||||
// upload file
|
||||
await Promise.all(
|
||||
files.map(async ({ fileId, file }) => {
|
||||
const { fileId: uploadFileId } = await uploadFile2DB({
|
||||
file,
|
||||
bucketName: BucketNameEnum.dataset,
|
||||
data: {
|
||||
datasetId
|
||||
},
|
||||
percentListen: (e) => {
|
||||
setSelectFiles((state) =>
|
||||
state.map((item) =>
|
||||
item.id === fileId
|
||||
? {
|
||||
...item,
|
||||
uploadedFileRate: item.uploadedFileRate
|
||||
? Math.max(e, item.uploadedFileRate)
|
||||
: e
|
||||
}
|
||||
: item
|
||||
)
|
||||
);
|
||||
}
|
||||
});
|
||||
setSelectFiles((state) =>
|
||||
state.map((item) =>
|
||||
item.id === fileId
|
||||
? {
|
||||
...item,
|
||||
dbFileId: uploadFileId,
|
||||
isUploading: false,
|
||||
uploadedFileRate: 100
|
||||
}
|
||||
: item
|
||||
)
|
||||
);
|
||||
})
|
||||
);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
onFinishSelect();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const selectFileCallback = useCallback(
|
||||
(files: SelectFileItemType[]) => {
|
||||
if (selectFiles.length + files.length > maxCount) {
|
||||
files = files.slice(0, maxCount - selectFiles.length);
|
||||
toast({
|
||||
status: 'warning',
|
||||
title: fileT('some_file_count_exceeds_limit', { maxCount })
|
||||
});
|
||||
}
|
||||
// size check
|
||||
if (!maxSize) {
|
||||
return onSelectFile(files);
|
||||
}
|
||||
const filterFiles = files.filter((item) => item.file.size <= maxSize);
|
||||
|
||||
if (filterFiles.length < files.length) {
|
||||
toast({
|
||||
status: 'warning',
|
||||
title: fileT('some_file_size_exceeds_limit', { maxSize: formatFileSize(maxSize) })
|
||||
});
|
||||
}
|
||||
|
||||
return onSelectFile(filterFiles);
|
||||
},
|
||||
[fileT, maxCount, maxSize, onSelectFile, selectFiles.length, toast]
|
||||
);
|
||||
|
||||
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(true);
|
||||
};
|
||||
|
||||
const handleDragLeave = (e: DragEvent<HTMLDivElement>) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(false);
|
||||
};
|
||||
|
||||
const handleDrop = async (e: DragEvent<HTMLDivElement>) => {
|
||||
e.preventDefault();
|
||||
setIsDragging(false);
|
||||
|
||||
const items = e.dataTransfer.items;
|
||||
const fileList: SelectFileItemType[] = [];
|
||||
|
||||
const firstEntry = items[0].webkitGetAsEntry();
|
||||
|
||||
if (firstEntry?.isDirectory && items.length === 1) {
|
||||
{
|
||||
const readFile = (entry: any) => {
|
||||
return new Promise((resolve) => {
|
||||
entry.file((file: File) => {
|
||||
const folderPath = (entry.fullPath || '').split('/').slice(2, -1).join('/');
|
||||
|
||||
if (filterTypeReg.test(file.name)) {
|
||||
fileList.push({
|
||||
fileId: getNanoid(),
|
||||
folderPath,
|
||||
file
|
||||
});
|
||||
}
|
||||
resolve(file);
|
||||
});
|
||||
});
|
||||
};
|
||||
const traverseFileTree = (dirReader: any) => {
|
||||
return new Promise((resolve) => {
|
||||
let fileNum = 0;
|
||||
dirReader.readEntries(async (entries: any[]) => {
|
||||
for await (const entry of entries) {
|
||||
if (entry.isFile) {
|
||||
await readFile(entry);
|
||||
fileNum++;
|
||||
} else if (entry.isDirectory) {
|
||||
await traverseFileTree(entry.createReader());
|
||||
}
|
||||
}
|
||||
|
||||
// chrome: readEntries will return 100 entries at most
|
||||
if (fileNum === 100) {
|
||||
await traverseFileTree(dirReader);
|
||||
}
|
||||
resolve('');
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
for await (const item of items) {
|
||||
const entry = item.webkitGetAsEntry();
|
||||
if (entry) {
|
||||
if (entry.isFile) {
|
||||
await readFile(entry);
|
||||
} else if (entry.isDirectory) {
|
||||
//@ts-ignore
|
||||
await traverseFileTree(entry.createReader());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (firstEntry?.isFile) {
|
||||
const files = Array.from(e.dataTransfer.files);
|
||||
let isErr = files.some((item) => item.type === '');
|
||||
if (isErr) {
|
||||
return toast({
|
||||
title: t('file:upload_error_description'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
|
||||
fileList.push(
|
||||
...files
|
||||
.filter((item) => filterTypeReg.test(item.name))
|
||||
.map((file) => ({
|
||||
fileId: getNanoid(),
|
||||
folderPath: '',
|
||||
file
|
||||
}))
|
||||
);
|
||||
} else {
|
||||
return toast({
|
||||
title: fileT('upload_error_description'),
|
||||
status: 'error'
|
||||
});
|
||||
}
|
||||
|
||||
selectFileCallback(fileList.slice(0, maxCount));
|
||||
};
|
||||
|
||||
return (
|
||||
<MyBox
|
||||
isLoading={isLoading}
|
||||
display={'flex'}
|
||||
flexDirection={'column'}
|
||||
alignItems={'center'}
|
||||
justifyContent={'center'}
|
||||
px={3}
|
||||
py={[4, 7]}
|
||||
borderWidth={'1.5px'}
|
||||
borderStyle={'dashed'}
|
||||
borderRadius={'md'}
|
||||
{...(isMaxSelected
|
||||
? {}
|
||||
: {
|
||||
cursor: 'pointer',
|
||||
_hover: {
|
||||
bg: 'primary.50',
|
||||
borderColor: 'primary.600'
|
||||
},
|
||||
borderColor: isDragging ? 'primary.600' : 'borderColor.high',
|
||||
onDragEnter: handleDragEnter,
|
||||
onDragOver: (e) => e.preventDefault(),
|
||||
onDragLeave: handleDragLeave,
|
||||
onDrop: handleDrop,
|
||||
onClick: onOpen
|
||||
})}
|
||||
{...props}
|
||||
>
|
||||
<MyIcon name={'common/uploadFileFill'} w={'32px'} />
|
||||
{isMaxSelected ? (
|
||||
<>
|
||||
<Box color={'myGray.500'} fontSize={'xs'}>
|
||||
{t('file:reached_max_file_count')}
|
||||
</Box>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Box fontWeight={'bold'}>
|
||||
{isDragging
|
||||
? fileT('release_the_mouse_to_upload_the_file')
|
||||
: fileT('select_and_drag_file_tip')}
|
||||
</Box>
|
||||
{/* file type */}
|
||||
<Box color={'myGray.500'} fontSize={'xs'}>
|
||||
{fileT('support_file_type', { fileType })}
|
||||
</Box>
|
||||
<Box color={'myGray.500'} fontSize={'xs'}>
|
||||
{/* max count */}
|
||||
{maxCount && fileT('support_max_count', { maxCount })}
|
||||
{/* max size */}
|
||||
{maxSize && fileT('support_max_size', { maxSize: formatFileSize(maxSize) })}
|
||||
</Box>
|
||||
|
||||
<File
|
||||
onSelect={(files) =>
|
||||
selectFileCallback(
|
||||
files.map((file) => ({
|
||||
fileId: getNanoid(),
|
||||
folderPath: '',
|
||||
file
|
||||
}))
|
||||
)
|
||||
}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</MyBox>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(FileSelector);
|
||||
@@ -0,0 +1,65 @@
|
||||
import React, { useState } from 'react';
|
||||
import MyModal from '@fastgpt/web/components/common/MyModal';
|
||||
import { ModalBody, ModalFooter, Button } from '@chakra-ui/react';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
|
||||
import { useRouter } from 'next/router';
|
||||
import { TabEnum } from '../../../../../pages/dataset/detail';
|
||||
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
|
||||
const FileModeSelector = ({ onClose }: { onClose: () => void }) => {
|
||||
const { t } = useTranslation();
|
||||
const router = useRouter();
|
||||
const [value, setValue] = useState<ImportDataSourceEnum>(ImportDataSourceEnum.fileLocal);
|
||||
|
||||
return (
|
||||
<MyModal
|
||||
isOpen
|
||||
onClose={onClose}
|
||||
iconSrc="modal/selectSource"
|
||||
title={t('common:core.dataset.import.Select source')}
|
||||
w={'600px'}
|
||||
>
|
||||
<ModalBody px={6} py={4}>
|
||||
<LeftRadio
|
||||
list={[
|
||||
{
|
||||
title: t('common:core.dataset.import.Local file'),
|
||||
desc: t('common:core.dataset.import.Local file desc'),
|
||||
value: ImportDataSourceEnum.fileLocal
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Web link'),
|
||||
desc: t('common:core.dataset.import.Web link desc'),
|
||||
value: ImportDataSourceEnum.fileLink
|
||||
},
|
||||
{
|
||||
title: t('common:core.dataset.import.Custom text'),
|
||||
desc: t('common:core.dataset.import.Custom text desc'),
|
||||
value: ImportDataSourceEnum.fileCustom
|
||||
}
|
||||
]}
|
||||
value={value}
|
||||
onChange={setValue}
|
||||
/>
|
||||
</ModalBody>
|
||||
<ModalFooter>
|
||||
<Button
|
||||
onClick={() =>
|
||||
router.replace({
|
||||
query: {
|
||||
...router.query,
|
||||
currentTab: TabEnum.import,
|
||||
source: value
|
||||
}
|
||||
})
|
||||
}
|
||||
>
|
||||
{t('common:common.Confirm')}
|
||||
</Button>
|
||||
</ModalFooter>
|
||||
</MyModal>
|
||||
);
|
||||
};
|
||||
|
||||
export default FileModeSelector;
|
||||
@@ -0,0 +1,102 @@
|
||||
import React, { useState } from 'react';
|
||||
import { Box, Flex, Grid, IconButton } from '@chakra-ui/react';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
|
||||
import MyMenu from '@fastgpt/web/components/common/MyMenu';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
const PreviewRawText = dynamic(() => import('./PreviewRawText'));
|
||||
const PreviewChunks = dynamic(() => import('./PreviewChunks'));
|
||||
|
||||
const Preview = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => {
|
||||
const { t } = useTranslation();
|
||||
|
||||
const { sources } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const [previewRawTextSource, setPreviewRawTextSource] = useState<ImportSourceItemType>();
|
||||
const [previewChunkSource, setPreviewChunkSource] = useState<ImportSourceItemType>();
|
||||
|
||||
return (
|
||||
<Box h={'100%'} w={'100%'} display={['block', 'flex']} flexDirection={'column'}>
|
||||
<Flex alignItems={'center'}>
|
||||
<MyIcon name={'core/dataset/fileCollection'} w={'20px'} />
|
||||
<Box fontSize={'md'}>{t('common:core.dataset.import.Sources list')}</Box>
|
||||
</Flex>
|
||||
<Box mt={3} flex={'1 0 0'} h={['auto', 0]} width={'100%'} overflowY={'auto'}>
|
||||
<Grid w={'100%'} gap={3} gridTemplateColumns={['1fr', '1fr', '1fr', '1fr', '1fr 1fr']}>
|
||||
{sources.map((source) => (
|
||||
<Flex
|
||||
key={source.id}
|
||||
bg={'white'}
|
||||
p={4}
|
||||
borderRadius={'md'}
|
||||
borderWidth={'1px'}
|
||||
borderColor={'borderColor.low'}
|
||||
boxShadow={'2'}
|
||||
alignItems={'center'}
|
||||
>
|
||||
<MyIcon name={source.icon as any} w={['1rem', '1.25rem']} />
|
||||
<Box mx={1} flex={'1 0 0'} wordBreak={'break-all'} fontSize={'sm'}>
|
||||
{source.sourceName}
|
||||
</Box>
|
||||
{showPreviewChunks && (
|
||||
<Box fontSize={'xs'} color={'myGray.600'}>
|
||||
<MyMenu
|
||||
Button={
|
||||
<IconButton
|
||||
icon={<MyIcon name={'common/viewLight'} w={'14px'} p={2} />}
|
||||
aria-label={''}
|
||||
size={'sm'}
|
||||
variant={'whitePrimary'}
|
||||
/>
|
||||
}
|
||||
menuList={[
|
||||
{
|
||||
children: [
|
||||
{
|
||||
label: (
|
||||
<Flex alignItems={'center'}>
|
||||
<MyIcon name={'core/dataset/fileCollection'} w={'14px'} mr={2} />
|
||||
{t('common:core.dataset.import.Preview raw text')}
|
||||
</Flex>
|
||||
),
|
||||
onClick: () => setPreviewRawTextSource(source)
|
||||
},
|
||||
{
|
||||
label: (
|
||||
<Flex alignItems={'center'}>
|
||||
<MyIcon name={'core/dataset/splitLight'} w={'14px'} mr={2} />
|
||||
{t('common:core.dataset.import.Preview chunks')}
|
||||
</Flex>
|
||||
),
|
||||
onClick: () => setPreviewChunkSource(source)
|
||||
}
|
||||
]
|
||||
}
|
||||
]}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
</Flex>
|
||||
))}
|
||||
</Grid>
|
||||
</Box>
|
||||
{!!previewRawTextSource && (
|
||||
<PreviewRawText
|
||||
previewSource={previewRawTextSource}
|
||||
onClose={() => setPreviewRawTextSource(undefined)}
|
||||
/>
|
||||
)}
|
||||
{!!previewChunkSource && (
|
||||
<PreviewChunks
|
||||
previewSource={previewChunkSource}
|
||||
onClose={() => setPreviewChunkSource(undefined)}
|
||||
/>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(Preview);
|
||||
@@ -0,0 +1,102 @@
|
||||
import React from 'react';
|
||||
import { Box } from '@chakra-ui/react';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type';
|
||||
import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer';
|
||||
import { getPreviewChunks } from '@/web/core/dataset/api';
|
||||
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
|
||||
import { getPreviewSourceReadType } from '../utils';
|
||||
|
||||
const PreviewChunks = ({
|
||||
previewSource,
|
||||
onClose
|
||||
}: {
|
||||
previewSource: ImportSourceItemType;
|
||||
onClose: () => void;
|
||||
}) => {
|
||||
const { importSource, chunkSize, chunkOverlapRatio, processParamsForm } = useContextSelector(
|
||||
DatasetImportContext,
|
||||
(v) => v
|
||||
);
|
||||
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
|
||||
|
||||
const { data = [], loading: isLoading } = useRequest2(
|
||||
async () => {
|
||||
if (importSource === ImportDataSourceEnum.fileCustom) {
|
||||
const customSplitChar = processParamsForm.getValues('customSplitChar');
|
||||
const { chunks } = splitText2Chunks({
|
||||
text: previewSource.rawText || '',
|
||||
chunkLen: chunkSize,
|
||||
overlapRatio: chunkOverlapRatio,
|
||||
customReg: customSplitChar ? [customSplitChar] : []
|
||||
});
|
||||
return chunks.map((chunk) => ({
|
||||
q: chunk,
|
||||
a: ''
|
||||
}));
|
||||
}
|
||||
|
||||
return getPreviewChunks({
|
||||
datasetId,
|
||||
type: getPreviewSourceReadType(previewSource),
|
||||
sourceId:
|
||||
previewSource.dbFileId ||
|
||||
previewSource.link ||
|
||||
previewSource.externalFileUrl ||
|
||||
previewSource.apiFileId ||
|
||||
'',
|
||||
|
||||
chunkSize,
|
||||
overlapRatio: chunkOverlapRatio,
|
||||
customSplitChar: processParamsForm.getValues('customSplitChar'),
|
||||
|
||||
selector: processParamsForm.getValues('webSelector'),
|
||||
isQAImport: importSource === ImportDataSourceEnum.csvTable,
|
||||
externalFileId: previewSource.externalFileId
|
||||
});
|
||||
},
|
||||
{
|
||||
manual: false
|
||||
}
|
||||
);
|
||||
|
||||
return (
|
||||
<MyRightDrawer
|
||||
onClose={onClose}
|
||||
iconSrc={previewSource.icon}
|
||||
title={previewSource.sourceName}
|
||||
isLoading={isLoading}
|
||||
maxW={['90vw', '40vw']}
|
||||
px={0}
|
||||
>
|
||||
<Box overflowY={'auto'} px={5} fontSize={'sm'}>
|
||||
{data.map((item, index) => (
|
||||
<Box
|
||||
key={index}
|
||||
whiteSpace={'pre-wrap'}
|
||||
fontSize={'sm'}
|
||||
p={4}
|
||||
bg={index % 2 === 0 ? 'white' : 'myWhite.600'}
|
||||
mb={3}
|
||||
borderRadius={'md'}
|
||||
borderWidth={'1px'}
|
||||
borderColor={'borderColor.low'}
|
||||
boxShadow={'2'}
|
||||
_notLast={{
|
||||
mb: 2
|
||||
}}
|
||||
>
|
||||
<Box color={'myGray.900'}>{item.q}</Box>
|
||||
<Box color={'myGray.500'}>{item.a}</Box>
|
||||
</Box>
|
||||
))}
|
||||
</Box>
|
||||
</MyRightDrawer>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(PreviewChunks);
|
||||
@@ -0,0 +1,78 @@
|
||||
import React from 'react';
|
||||
import { Box } from '@chakra-ui/react';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type';
|
||||
import { getPreviewFileContent } from '@/web/common/file/api';
|
||||
import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer';
|
||||
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { useToast } from '@fastgpt/web/hooks/useToast';
|
||||
import { getErrText } from '@fastgpt/global/common/error/utils';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
import { getPreviewSourceReadType } from '../utils';
|
||||
|
||||
const PreviewRawText = ({
|
||||
previewSource,
|
||||
onClose
|
||||
}: {
|
||||
previewSource: ImportSourceItemType;
|
||||
onClose: () => void;
|
||||
}) => {
|
||||
const { toast } = useToast();
|
||||
const { importSource, processParamsForm } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
|
||||
|
||||
const { data, loading: isLoading } = useRequest2(
|
||||
async () => {
|
||||
if (importSource === ImportDataSourceEnum.fileCustom && previewSource.rawText) {
|
||||
return {
|
||||
previewContent: previewSource.rawText.slice(0, 3000)
|
||||
};
|
||||
}
|
||||
|
||||
return getPreviewFileContent({
|
||||
datasetId,
|
||||
type: getPreviewSourceReadType(previewSource),
|
||||
sourceId:
|
||||
previewSource.dbFileId ||
|
||||
previewSource.link ||
|
||||
previewSource.externalFileUrl ||
|
||||
previewSource.apiFileId ||
|
||||
'',
|
||||
|
||||
isQAImport: importSource === ImportDataSourceEnum.csvTable,
|
||||
selector: processParamsForm.getValues('webSelector'),
|
||||
externalFileId: previewSource.externalFileId
|
||||
});
|
||||
},
|
||||
{
|
||||
refreshDeps: [previewSource.dbFileId, previewSource.link, previewSource.externalFileUrl],
|
||||
manual: false,
|
||||
onError(err) {
|
||||
toast({
|
||||
status: 'warning',
|
||||
title: getErrText(err)
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const rawText = data?.previewContent || '';
|
||||
|
||||
return (
|
||||
<MyRightDrawer
|
||||
onClose={onClose}
|
||||
iconSrc={previewSource.icon}
|
||||
title={previewSource.sourceName}
|
||||
isLoading={isLoading}
|
||||
px={0}
|
||||
>
|
||||
<Box whiteSpace={'pre-wrap'} overflowY={'auto'} px={5} fontSize={'sm'}>
|
||||
{rawText}
|
||||
</Box>
|
||||
</MyRightDrawer>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(PreviewRawText);
|
||||
@@ -0,0 +1,123 @@
|
||||
import React, { useState } from 'react';
|
||||
import {
|
||||
Flex,
|
||||
TableContainer,
|
||||
Table,
|
||||
Thead,
|
||||
Tr,
|
||||
Th,
|
||||
Td,
|
||||
Tbody,
|
||||
Progress,
|
||||
IconButton
|
||||
} from '@chakra-ui/react';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useI18n } from '@/web/context/I18n';
|
||||
|
||||
const PreviewRawText = dynamic(() => import('./PreviewRawText'));
|
||||
|
||||
export const RenderUploadFiles = ({
|
||||
files,
|
||||
setFiles,
|
||||
showPreviewContent
|
||||
}: {
|
||||
files: ImportSourceItemType[];
|
||||
setFiles: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
|
||||
showPreviewContent?: boolean;
|
||||
}) => {
|
||||
const { t } = useTranslation();
|
||||
const { fileT } = useI18n();
|
||||
const [previewFile, setPreviewFile] = useState<ImportSourceItemType>();
|
||||
|
||||
return files.length > 0 ? (
|
||||
<>
|
||||
<TableContainer mt={5}>
|
||||
<Table variant={'simple'} fontSize={'sm'} draggable={false}>
|
||||
<Thead draggable={false}>
|
||||
<Tr bg={'myGray.100'} mb={2}>
|
||||
<Th borderLeftRadius={'md'} borderBottom={'none'} py={4}>
|
||||
{fileT('file_name')}
|
||||
</Th>
|
||||
<Th borderBottom={'none'} py={4}>
|
||||
{t('common:core.dataset.import.Upload file progress')}
|
||||
</Th>
|
||||
<Th borderBottom={'none'} py={4}>
|
||||
{fileT('file_size')}
|
||||
</Th>
|
||||
<Th borderRightRadius={'md'} borderBottom={'none'} py={4}>
|
||||
{t('common:common.Action')}
|
||||
</Th>
|
||||
</Tr>
|
||||
</Thead>
|
||||
<Tbody>
|
||||
{files.map((item) => (
|
||||
<Tr key={item.id}>
|
||||
<Td>
|
||||
<Flex alignItems={'center'}>
|
||||
<MyIcon name={item.icon as any} w={'16px'} mr={1} />
|
||||
{item.sourceName}
|
||||
</Flex>
|
||||
</Td>
|
||||
<Td>
|
||||
<Flex alignItems={'center'} fontSize={'xs'}>
|
||||
<Progress
|
||||
value={item.uploadedFileRate}
|
||||
h={'6px'}
|
||||
w={'100%'}
|
||||
maxW={'210px'}
|
||||
size="sm"
|
||||
borderRadius={'20px'}
|
||||
colorScheme={(item.uploadedFileRate || 0) >= 100 ? 'green' : 'blue'}
|
||||
bg="myGray.200"
|
||||
hasStripe
|
||||
isAnimated
|
||||
mr={2}
|
||||
/>
|
||||
{`${item.uploadedFileRate}%`}
|
||||
</Flex>
|
||||
</Td>
|
||||
<Td>{item.sourceSize}</Td>
|
||||
<Td>
|
||||
{!item.isUploading && (
|
||||
<Flex alignItems={'center'} gap={4}>
|
||||
{showPreviewContent && (
|
||||
<MyTooltip label={t('common:core.dataset.import.Preview raw text')}>
|
||||
<IconButton
|
||||
variant={'whitePrimary'}
|
||||
size={'sm'}
|
||||
icon={<MyIcon name={'common/viewLight'} w={'18px'} />}
|
||||
aria-label={''}
|
||||
onClick={() => setPreviewFile(item)}
|
||||
/>
|
||||
</MyTooltip>
|
||||
)}
|
||||
|
||||
<IconButton
|
||||
variant={'grayDanger'}
|
||||
size={'sm'}
|
||||
icon={<MyIcon name={'delete'} w={'14px'} />}
|
||||
aria-label={''}
|
||||
onClick={() => {
|
||||
setFiles((state) => state.filter((file) => file.id !== item.id));
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
)}
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
</Tbody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
{!!previewFile && (
|
||||
<PreviewRawText previewSource={previewFile} onClose={() => setPreviewFile(undefined)} />
|
||||
)}
|
||||
</>
|
||||
) : null;
|
||||
};
|
||||
|
||||
export default RenderUploadFiles;
|
||||
@@ -0,0 +1,283 @@
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
import React, { useCallback, useMemo, useState } from 'react';
|
||||
import dynamic from 'next/dynamic';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { Box, Button, Checkbox, Flex } from '@chakra-ui/react';
|
||||
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
import { getApiDatasetFileList, getApiDatasetFileListExistId } from '@/web/core/dataset/api';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
|
||||
import FolderPath from '@/components/common/folder/Path';
|
||||
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
|
||||
import MyBox from '@fastgpt/web/components/common/MyBox';
|
||||
import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
|
||||
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
|
||||
import { useMount } from 'ahooks';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const APIDatasetCollection = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <CustomAPIFileInput />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks={true} />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(APIDatasetCollection);
|
||||
|
||||
const CustomAPIFileInput = () => {
|
||||
const { t } = useTranslation();
|
||||
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
|
||||
const goToNext = useContextSelector(DatasetImportContext, (v) => v.goToNext);
|
||||
|
||||
const sources = useContextSelector(DatasetImportContext, (v) => v.sources);
|
||||
const setSources = useContextSelector(DatasetImportContext, (v) => v.setSources);
|
||||
|
||||
const [selectFiles, setSelectFiles] = useState<APIFileItem[]>([]);
|
||||
const [parent, setParent] = useState<ParentTreePathItemType>({
|
||||
parentId: '',
|
||||
parentName: ''
|
||||
});
|
||||
const [paths, setPaths] = useState<ParentTreePathItemType[]>([]);
|
||||
|
||||
const [searchKey, setSearchKey] = useState('');
|
||||
|
||||
const { data: fileList = [], loading } = useRequest2(
|
||||
async () => {
|
||||
return getApiDatasetFileList({
|
||||
datasetId: datasetDetail._id,
|
||||
parentId: parent?.parentId,
|
||||
searchKey: searchKey
|
||||
});
|
||||
},
|
||||
{
|
||||
refreshDeps: [datasetDetail._id, datasetDetail.apiServer, parent, searchKey],
|
||||
throttleWait: 500,
|
||||
manual: false
|
||||
}
|
||||
);
|
||||
|
||||
const { data: existIdList = [] } = useRequest2(
|
||||
() => getApiDatasetFileListExistId({ datasetId: datasetDetail._id }),
|
||||
{
|
||||
manual: false
|
||||
}
|
||||
);
|
||||
|
||||
// Init selected files
|
||||
useMount(() => {
|
||||
setSelectFiles(sources.map((item) => item.apiFile).filter(Boolean) as APIFileItem[]);
|
||||
});
|
||||
|
||||
const { runAsync: onclickNext, loading: onNextLoading } = useRequest2(
|
||||
async () => {
|
||||
// Computed all selected files
|
||||
const getFilesRecursively = async (files: APIFileItem[]): Promise<APIFileItem[]> => {
|
||||
const allFiles: APIFileItem[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
if (file.type === 'folder') {
|
||||
const folderFiles = await getApiDatasetFileList({
|
||||
datasetId: datasetDetail._id,
|
||||
parentId: file?.id
|
||||
});
|
||||
|
||||
const subFiles = await getFilesRecursively(folderFiles);
|
||||
allFiles.push(...subFiles);
|
||||
} else {
|
||||
allFiles.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
return allFiles;
|
||||
};
|
||||
|
||||
const allFiles = await getFilesRecursively(selectFiles);
|
||||
|
||||
setSources(
|
||||
allFiles
|
||||
.filter((item) => !existIdList.includes(item.id))
|
||||
.map((item) => ({
|
||||
id: item.id,
|
||||
apiFileId: item.id,
|
||||
apiFile: item,
|
||||
createStatus: 'waiting',
|
||||
sourceName: item.name,
|
||||
icon: getSourceNameIcon({ sourceName: item.name }) as any
|
||||
}))
|
||||
);
|
||||
},
|
||||
{
|
||||
onSuccess() {
|
||||
goToNext();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const handleItemClick = useCallback(
|
||||
(item: APIFileItem) => {
|
||||
if (item.hasChild) {
|
||||
setPaths((state) => [...state, { parentId: item.id, parentName: item.name }]);
|
||||
return setParent({
|
||||
parentId: item.id,
|
||||
parentName: item.name
|
||||
});
|
||||
}
|
||||
|
||||
const isCurrentlySelected = selectFiles.some((file) => file.id === item.id);
|
||||
if (isCurrentlySelected) {
|
||||
setSelectFiles((state) => state.filter((file) => file.id !== item.id));
|
||||
} else {
|
||||
setSelectFiles((state) => [...state, item]);
|
||||
}
|
||||
},
|
||||
[selectFiles]
|
||||
);
|
||||
|
||||
const handleSelectAll = useCallback(() => {
|
||||
const isAllSelected = fileList.length === selectFiles.length;
|
||||
|
||||
if (isAllSelected) {
|
||||
setSelectFiles([]);
|
||||
} else {
|
||||
setSelectFiles(fileList);
|
||||
}
|
||||
}, [fileList, selectFiles]);
|
||||
|
||||
return (
|
||||
<MyBox isLoading={loading} position="relative" h="full">
|
||||
<Flex flexDirection={'column'} h="full">
|
||||
<Flex justifyContent={'space-between'}>
|
||||
<FolderPath
|
||||
paths={paths}
|
||||
onClick={(parentId) => {
|
||||
const index = paths.findIndex((item) => item.parentId === parentId);
|
||||
|
||||
setParent(paths[index]);
|
||||
setPaths(paths.slice(0, index + 1));
|
||||
}}
|
||||
/>
|
||||
{datasetDetail.apiServer && (
|
||||
<Box w={'240px'}>
|
||||
<SearchInput
|
||||
value={searchKey}
|
||||
onChange={(e) => setSearchKey(e.target.value)}
|
||||
placeholder={t('common:core.workflow.template.Search')}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
</Flex>
|
||||
<Box flex={1} overflowY="auto" mb={16}>
|
||||
<Box ml={2} mt={3}>
|
||||
<Flex
|
||||
alignItems={'center'}
|
||||
py={3}
|
||||
cursor={'pointer'}
|
||||
bg={'myGray.50'}
|
||||
pl={7}
|
||||
rounded={'8px'}
|
||||
fontSize={'sm'}
|
||||
fontWeight={'medium'}
|
||||
color={'myGray.900'}
|
||||
onClick={(e) => {
|
||||
if (!(e.target as HTMLElement).closest('.checkbox')) {
|
||||
handleSelectAll();
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Checkbox
|
||||
className="checkbox"
|
||||
mr={2}
|
||||
isChecked={fileList.length === selectFiles.length}
|
||||
onChange={handleSelectAll}
|
||||
/>
|
||||
{t('common:Select_all')}
|
||||
</Flex>
|
||||
{fileList.map((item) => {
|
||||
const isFolder = item.type === 'folder';
|
||||
const isExists = existIdList.includes(item.id);
|
||||
const isChecked = isExists || selectFiles.some((file) => file.id === item.id);
|
||||
|
||||
return (
|
||||
<Flex
|
||||
key={item.id}
|
||||
py={3}
|
||||
_hover={{ bg: 'primary.50' }}
|
||||
pl={7}
|
||||
cursor={'pointer'}
|
||||
onClick={(e) => {
|
||||
if (isExists) return;
|
||||
if (!(e.target as HTMLElement).closest('.checkbox')) {
|
||||
handleItemClick(item);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Checkbox
|
||||
className="checkbox"
|
||||
mr={2.5}
|
||||
isChecked={isChecked}
|
||||
isDisabled={isExists}
|
||||
onChange={(e) => {
|
||||
e.stopPropagation();
|
||||
if (isExists) return;
|
||||
if (isChecked) {
|
||||
setSelectFiles((state) => state.filter((file) => file.id !== item.id));
|
||||
} else {
|
||||
setSelectFiles((state) => [...state, item]);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<MyIcon
|
||||
name={
|
||||
!isFolder
|
||||
? (getSourceNameIcon({ sourceName: item.name }) as any)
|
||||
: 'common/folderFill'
|
||||
}
|
||||
w={'18px'}
|
||||
mr={1.5}
|
||||
/>
|
||||
<Box fontSize={'sm'} fontWeight={'medium'} color={'myGray.900'}>
|
||||
{item.name}
|
||||
</Box>
|
||||
{item.hasChild && <MyIcon name="core/chat/chevronRight" w={'18px'} ml={2} />}
|
||||
</Flex>
|
||||
);
|
||||
})}
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
<Box
|
||||
position="absolute"
|
||||
display={'flex'}
|
||||
justifyContent={'end'}
|
||||
bottom={0}
|
||||
left={0}
|
||||
right={0}
|
||||
p={4}
|
||||
>
|
||||
<Button
|
||||
isDisabled={selectFiles.length === 0}
|
||||
isLoading={onNextLoading}
|
||||
onClick={onclickNext}
|
||||
>
|
||||
{selectFiles.length > 0
|
||||
? `${t('common:core.dataset.import.Total files', { total: selectFiles.length })} | `
|
||||
: ''}
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Box>
|
||||
</Flex>
|
||||
</MyBox>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,186 @@
|
||||
import React, { useEffect } from 'react';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { useFieldArray, useForm } from 'react-hook-form';
|
||||
import {
|
||||
Box,
|
||||
Button,
|
||||
Flex,
|
||||
Table,
|
||||
Thead,
|
||||
Tbody,
|
||||
Tr,
|
||||
Th,
|
||||
Td,
|
||||
TableContainer,
|
||||
Input
|
||||
} from '@chakra-ui/react';
|
||||
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
import { getFileIcon } from '@fastgpt/global/common/file/icon';
|
||||
import { SmallAddIcon } from '@chakra-ui/icons';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const ExternalFileCollection = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <CustomLinkInput />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks={true} />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(ExternalFileCollection);
|
||||
|
||||
const CustomLinkInput = () => {
|
||||
const { t } = useTranslation();
|
||||
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const { register, reset, handleSubmit, control } = useForm<{
|
||||
list: {
|
||||
sourceName: string;
|
||||
externalFileUrl: string;
|
||||
externalFileId: string;
|
||||
}[];
|
||||
}>({
|
||||
defaultValues: {
|
||||
list: [
|
||||
{
|
||||
sourceName: '',
|
||||
externalFileUrl: '',
|
||||
externalFileId: ''
|
||||
}
|
||||
]
|
||||
}
|
||||
});
|
||||
|
||||
const {
|
||||
fields: list,
|
||||
append,
|
||||
remove,
|
||||
update
|
||||
} = useFieldArray({
|
||||
control,
|
||||
name: 'list'
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (sources.length > 0) {
|
||||
reset({
|
||||
list: sources.map((item) => ({
|
||||
sourceName: item.sourceName,
|
||||
externalFileUrl: item.externalFileUrl || '',
|
||||
externalFileId: item.externalFileId || ''
|
||||
}))
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<Box>
|
||||
<TableContainer>
|
||||
<Table bg={'white'}>
|
||||
<Thead>
|
||||
<Tr bg={'myGray.50'}>
|
||||
<Th>{t('dataset:external_url')}</Th>
|
||||
<Th>{t('dataset:external_id')}</Th>
|
||||
<Th>{t('dataset:filename')}</Th>
|
||||
<Th></Th>
|
||||
</Tr>
|
||||
</Thead>
|
||||
<Tbody>
|
||||
{list.map((item, index) => (
|
||||
<Tr key={item.id}>
|
||||
<Td>
|
||||
<Input
|
||||
{...register(`list.${index}.externalFileUrl`, {
|
||||
required: index !== list.length - 1,
|
||||
onBlur(e) {
|
||||
const val = (e.target.value || '') as string;
|
||||
if (val.includes('.') && !list[index]?.sourceName) {
|
||||
const sourceName = val.split('/').pop() || '';
|
||||
update(index, {
|
||||
...list[index],
|
||||
externalFileUrl: val,
|
||||
sourceName: decodeURIComponent(sourceName)
|
||||
});
|
||||
}
|
||||
if (val && index === list.length - 1) {
|
||||
append({
|
||||
sourceName: '',
|
||||
externalFileUrl: '',
|
||||
externalFileId: ''
|
||||
});
|
||||
}
|
||||
}
|
||||
})}
|
||||
/>
|
||||
</Td>
|
||||
<Td>
|
||||
<Input {...register(`list.${index}.externalFileId`)} />
|
||||
</Td>
|
||||
<Td>
|
||||
<Input {...register(`list.${index}.sourceName`)} />
|
||||
</Td>
|
||||
<Td>
|
||||
<MyIcon
|
||||
name={'delete'}
|
||||
w={'16px'}
|
||||
cursor={'pointer'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
onClick={() => remove(index)}
|
||||
/>
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
</Tbody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
<Flex mt={5} justifyContent={'space-between'}>
|
||||
<Button
|
||||
variant={'whitePrimary'}
|
||||
leftIcon={<SmallAddIcon />}
|
||||
onClick={() => {
|
||||
append({
|
||||
sourceName: '',
|
||||
externalFileUrl: '',
|
||||
externalFileId: ''
|
||||
});
|
||||
}}
|
||||
>
|
||||
{t('common:add_new')}
|
||||
</Button>
|
||||
<Button
|
||||
isDisabled={list.filter((item) => !!item.externalFileUrl).length === 0}
|
||||
onClick={handleSubmit((data) => {
|
||||
setSources(
|
||||
data.list
|
||||
.filter((item) => !!item.externalFileUrl)
|
||||
.map((item) => ({
|
||||
id: getNanoid(32),
|
||||
createStatus: 'waiting',
|
||||
sourceName: item.sourceName || item.externalFileUrl,
|
||||
icon: getFileIcon(item.externalFileUrl),
|
||||
externalFileId: item.externalFileId,
|
||||
externalFileUrl: item.externalFileUrl
|
||||
}))
|
||||
);
|
||||
|
||||
goToNext();
|
||||
})}
|
||||
>
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,106 @@
|
||||
import React, { useCallback, useEffect } from 'react';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { Box, Button, Flex, Input, Textarea } from '@chakra-ui/react';
|
||||
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const CustomTet = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <CustomTextInput />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(CustomTet);
|
||||
|
||||
const CustomTextInput = () => {
|
||||
const { t } = useTranslation();
|
||||
const { sources, goToNext, setSources } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const { register, reset, handleSubmit } = useForm({
|
||||
defaultValues: {
|
||||
name: '',
|
||||
value: ''
|
||||
}
|
||||
});
|
||||
|
||||
const onSubmit = useCallback(
|
||||
(data: { name: string; value: string }) => {
|
||||
const fileId = getNanoid(32);
|
||||
|
||||
setSources([
|
||||
{
|
||||
id: fileId,
|
||||
createStatus: 'waiting',
|
||||
rawText: data.value,
|
||||
sourceName: data.name,
|
||||
icon: 'file/fill/manual'
|
||||
}
|
||||
]);
|
||||
goToNext();
|
||||
},
|
||||
[goToNext, setSources]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const source = sources[0];
|
||||
if (source) {
|
||||
reset({
|
||||
name: source.sourceName,
|
||||
value: source.rawText
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<Box maxW={['100%', '800px']}>
|
||||
<Box display={['block', 'flex']} alignItems={'center'}>
|
||||
<Box flex={'0 0 120px'} fontSize={'sm'}>
|
||||
{t('common:core.dataset.collection.Collection name')}
|
||||
</Box>
|
||||
<Input
|
||||
flex={'1 0 0'}
|
||||
maxW={['100%', '350px']}
|
||||
{...register('name', {
|
||||
required: true
|
||||
})}
|
||||
placeholder={t('common:core.dataset.collection.Collection name')}
|
||||
bg={'myGray.50'}
|
||||
/>
|
||||
</Box>
|
||||
<Box display={['block', 'flex']} alignItems={'flex-start'} mt={5}>
|
||||
<Box flex={'0 0 120px'} fontSize={'sm'}>
|
||||
{t('common:core.dataset.collection.Collection raw text')}
|
||||
</Box>
|
||||
<Textarea
|
||||
flex={'1 0 0'}
|
||||
w={'100%'}
|
||||
rows={15}
|
||||
placeholder={t('common:core.dataset.collection.Collection raw text')}
|
||||
{...register('value', {
|
||||
required: true
|
||||
})}
|
||||
bg={'myGray.50'}
|
||||
/>
|
||||
</Box>
|
||||
<Flex mt={5} justifyContent={'flex-end'}>
|
||||
<Button onClick={handleSubmit((data) => onSubmit(data))}>
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,153 @@
|
||||
import React, { useEffect } from 'react';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { Box, Button, Flex, Input, Link, Textarea } from '@chakra-ui/react';
|
||||
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { LinkCollectionIcon } from '@fastgpt/global/core/dataset/constants';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import { getDocPath } from '@/web/common/system/doc';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const LinkCollection = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <CustomLinkImport />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(LinkCollection);
|
||||
|
||||
const CustomLinkImport = () => {
|
||||
const { t } = useTranslation();
|
||||
const { feConfigs } = useSystemStore();
|
||||
const { goToNext, sources, setSources, processParamsForm } = useContextSelector(
|
||||
DatasetImportContext,
|
||||
(v) => v
|
||||
);
|
||||
const { register, reset, handleSubmit, watch } = useForm({
|
||||
defaultValues: {
|
||||
link: ''
|
||||
}
|
||||
});
|
||||
|
||||
const link = watch('link');
|
||||
const linkList = link.split('\n').filter((item) => item);
|
||||
|
||||
useEffect(() => {
|
||||
reset({
|
||||
link: sources
|
||||
.map((item) => item.link)
|
||||
.filter((item) => item)
|
||||
.join('\n')
|
||||
});
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<Box maxW={['100%', '800px']}>
|
||||
<Box display={['block', 'flex']} alignItems={'flex-start'} mt={1}>
|
||||
<Box flex={'0 0 100px'} fontSize={'sm'}>
|
||||
{t('common:core.dataset.import.Link name')}
|
||||
</Box>
|
||||
<Textarea
|
||||
flex={'1 0 0'}
|
||||
w={'100%'}
|
||||
rows={10}
|
||||
placeholder={t('common:core.dataset.import.Link name placeholder')}
|
||||
bg={'myGray.50'}
|
||||
overflowX={'auto'}
|
||||
whiteSpace={'nowrap'}
|
||||
{...register('link', {
|
||||
required: true
|
||||
})}
|
||||
/>
|
||||
</Box>
|
||||
<Box display={['block', 'flex']} alignItems={'center'} mt={4}>
|
||||
<Box flex={'0 0 100px'} fontSize={'sm'}>
|
||||
{t('common:core.dataset.website.Selector')}
|
||||
<Box color={'myGray.500'} fontSize={'sm'}>
|
||||
{feConfigs?.docUrl && (
|
||||
<Link
|
||||
href={getDocPath('/docs/guide/knowledge_base/websync/#选择器如何使用')}
|
||||
target="_blank"
|
||||
>
|
||||
{t('common:core.dataset.website.Selector Course')}
|
||||
</Link>
|
||||
)}
|
||||
</Box>
|
||||
</Box>
|
||||
<Input
|
||||
flex={'1 0 0'}
|
||||
maxW={['100%', '350px']}
|
||||
{...processParamsForm.register('webSelector')}
|
||||
placeholder={'body .content #document'}
|
||||
bg={'myGray.50'}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
<Flex my={4} flexWrap={'wrap'} gap={4} alignItems={'center'} pl={[0, '100px']}>
|
||||
{linkList.map((item, i) => (
|
||||
<Flex
|
||||
key={`${item}-${i}`}
|
||||
alignItems={'center'}
|
||||
px={4}
|
||||
py={2}
|
||||
borderRadius={'md'}
|
||||
bg={'myGray.100'}
|
||||
>
|
||||
<MyIcon name={LinkCollectionIcon} w={'16px'} />
|
||||
<Box ml={1} mr={3} wordBreak={'break-all'}>
|
||||
{item}
|
||||
</Box>
|
||||
<MyIcon
|
||||
name={'common/closeLight'}
|
||||
w={'14px'}
|
||||
color={'myGray.500'}
|
||||
cursor={'pointer'}
|
||||
onClick={() => {
|
||||
const newLinkList = linkList.filter((link, index) => index !== i);
|
||||
reset({
|
||||
link: newLinkList.join('\n')
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
))}
|
||||
</Flex>
|
||||
<Flex mt={5} justifyContent={'flex-end'}>
|
||||
<Button
|
||||
onClick={handleSubmit((data) => {
|
||||
const newLinkList = data.link.split('\n').filter((item) => item);
|
||||
|
||||
setSources(
|
||||
newLinkList.map((link) => ({
|
||||
id: getNanoid(32),
|
||||
createStatus: 'waiting',
|
||||
link,
|
||||
sourceName: link,
|
||||
icon: LinkCollectionIcon
|
||||
}))
|
||||
);
|
||||
|
||||
goToNext();
|
||||
})}
|
||||
>
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,79 @@
|
||||
import React, { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
|
||||
import { Box, Button } from '@chakra-ui/react';
|
||||
import FileSelector from '../components/FileSelector';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { RenderUploadFiles } from '../components/RenderFiles';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const fileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx';
|
||||
|
||||
const FileLocal = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <SelectFile />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(FileLocal);
|
||||
|
||||
const SelectFile = React.memo(function SelectFile() {
|
||||
const { t } = useTranslation();
|
||||
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>(
|
||||
sources.map((source) => ({
|
||||
isUploading: false,
|
||||
...source
|
||||
}))
|
||||
);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
|
||||
|
||||
useEffect(() => {
|
||||
setSources(successFiles);
|
||||
}, [setSources, successFiles]);
|
||||
|
||||
const onclickNext = useCallback(() => {
|
||||
// filter uploaded files
|
||||
setSelectFiles((state) => state.filter((item) => item.dbFileId));
|
||||
goToNext();
|
||||
}, [goToNext]);
|
||||
|
||||
return (
|
||||
<Box>
|
||||
<FileSelector
|
||||
fileType={fileType}
|
||||
selectFiles={selectFiles}
|
||||
setSelectFiles={setSelectFiles}
|
||||
onStartSelect={() => setUploading(true)}
|
||||
onFinishSelect={() => setUploading(false)}
|
||||
/>
|
||||
|
||||
{/* render files */}
|
||||
<RenderUploadFiles files={selectFiles} setFiles={setSelectFiles} showPreviewContent />
|
||||
|
||||
<Box textAlign={'right'} mt={5}>
|
||||
<Button isDisabled={successFiles.length === 0 || uploading} onClick={onclickNext}>
|
||||
{selectFiles.length > 0
|
||||
? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
|
||||
: ''}
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
import React from 'react';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
import DataProcess from '../commonProgress/DataProcess';
|
||||
import { useRouter } from 'next/router';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
import { getDatasetCollectionById } from '@/web/core/dataset/api';
|
||||
import MyBox from '@fastgpt/web/components/common/MyBox';
|
||||
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
|
||||
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
|
||||
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const ReTraining = () => {
|
||||
const router = useRouter();
|
||||
|
||||
const { collectionId = '' } = router.query as {
|
||||
collectionId: string;
|
||||
};
|
||||
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
const setSources = useContextSelector(DatasetImportContext, (v) => v.setSources);
|
||||
const processParamsForm = useContextSelector(DatasetImportContext, (v) => v.processParamsForm);
|
||||
|
||||
const { loading } = useRequest2(() => getDatasetCollectionById(collectionId), {
|
||||
refreshDeps: [collectionId],
|
||||
manual: false,
|
||||
onSuccess: (collection) => {
|
||||
setSources([
|
||||
{
|
||||
dbFileId: collection.fileId,
|
||||
link: collection.rawLink,
|
||||
apiFileId: collection.apiFileId,
|
||||
|
||||
createStatus: 'waiting',
|
||||
icon: getCollectionIcon(collection.type, collection.name),
|
||||
id: collection._id,
|
||||
isUploading: false,
|
||||
sourceName: collection.name,
|
||||
uploadedFileRate: 100
|
||||
}
|
||||
]);
|
||||
processParamsForm.reset({
|
||||
mode: collection.trainingType,
|
||||
way: ImportProcessWayEnum.auto,
|
||||
embeddingChunkSize: collection.chunkSize,
|
||||
qaChunkSize: collection.chunkSize,
|
||||
customSplitChar: collection.chunkSplitter,
|
||||
qaPrompt: collection.qaPrompt,
|
||||
webSelector: collection.metadata?.webSelector
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<MyBox isLoading={loading} h={'100%'} overflow={'auto'}>
|
||||
{activeStep === 0 && <DataProcess showPreviewChunks={true} />}
|
||||
{activeStep === 1 && <Upload />}
|
||||
</MyBox>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(ReTraining);
|
||||
@@ -0,0 +1,101 @@
|
||||
import React, { useEffect, useMemo, useState } from 'react';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
|
||||
import { Box, Button } from '@chakra-ui/react';
|
||||
import FileSelector from '../components/FileSelector';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
import { fileDownload } from '@/web/common/file/utils';
|
||||
import { RenderUploadFiles } from '../components/RenderFiles';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const fileType = '.csv';
|
||||
|
||||
const FileLocal = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <SelectFile />}
|
||||
{activeStep === 1 && <PreviewData showPreviewChunks />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(FileLocal);
|
||||
|
||||
const csvTemplate = `index,content
|
||||
"第一列内容","第二列内容"
|
||||
"必填列","可选列。CSV 中请注意内容不能包含双引号，双引号是列分割符号"
|
||||
"只会将第一和第二列内容导入，其余列会被忽略",""
|
||||
"结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。",""
|
||||
"AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`;
|
||||
|
||||
const SelectFile = React.memo(function SelectFile() {
|
||||
const { t } = useTranslation();
|
||||
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>(
|
||||
sources.map((source) => ({
|
||||
isUploading: false,
|
||||
...source
|
||||
}))
|
||||
);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
|
||||
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
|
||||
|
||||
useEffect(() => {
|
||||
setSources(successFiles);
|
||||
}, [successFiles]);
|
||||
|
||||
return (
|
||||
<Box>
|
||||
<FileSelector
|
||||
fileType={fileType}
|
||||
selectFiles={selectFiles}
|
||||
setSelectFiles={setSelectFiles}
|
||||
onStartSelect={() => setUploading(true)}
|
||||
onFinishSelect={() => setUploading(false)}
|
||||
/>
|
||||
|
||||
<Box
|
||||
mt={4}
|
||||
color={'primary.600'}
|
||||
textDecoration={'underline'}
|
||||
cursor={'pointer'}
|
||||
onClick={() =>
|
||||
fileDownload({
|
||||
text: csvTemplate,
|
||||
type: 'text/csv;charset=utf-8',
|
||||
filename: 'template.csv'
|
||||
})
|
||||
}
|
||||
>
|
||||
{t('common:core.dataset.import.Down load csv template')}
|
||||
</Box>
|
||||
|
||||
{/* render files */}
|
||||
<RenderUploadFiles files={selectFiles} setFiles={setSelectFiles} />
|
||||
|
||||
<Box textAlign={'right'} mt={5}>
|
||||
<Button
|
||||
isDisabled={successFiles.length === 0 || uploading}
|
||||
onClick={() => {
|
||||
setSelectFiles((state) => state.filter((item) => !item.errorMsg));
|
||||
goToNext();
|
||||
}}
|
||||
>
|
||||
{selectFiles.length > 0
|
||||
? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
|
||||
: ''}
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import React, { useMemo } from 'react';
|
||||
import { Box, Flex } from '@chakra-ui/react';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import DatasetImportContextProvider, { DatasetImportContext } from './Context';
|
||||
|
||||
const FileLocal = dynamic(() => import('./diffSource/FileLocal'));
|
||||
const FileLink = dynamic(() => import('./diffSource/FileLink'));
|
||||
const FileCustomText = dynamic(() => import('./diffSource/FileCustomText'));
|
||||
const TableLocal = dynamic(() => import('./diffSource/TableLocal'));
|
||||
const ExternalFileCollection = dynamic(() => import('./diffSource/ExternalFile'));
|
||||
const APIDatasetCollection = dynamic(() => import('./diffSource/APIDataset'));
|
||||
const ReTraining = dynamic(() => import('./diffSource/ReTraining'));
|
||||
|
||||
const ImportDataset = () => {
|
||||
const importSource = useContextSelector(DatasetImportContext, (v) => v.importSource);
|
||||
|
||||
const ImportComponent = useMemo(() => {
|
||||
if (importSource === ImportDataSourceEnum.reTraining) return ReTraining;
|
||||
if (importSource === ImportDataSourceEnum.fileLocal) return FileLocal;
|
||||
if (importSource === ImportDataSourceEnum.fileLink) return FileLink;
|
||||
if (importSource === ImportDataSourceEnum.fileCustom) return FileCustomText;
|
||||
if (importSource === ImportDataSourceEnum.csvTable) return TableLocal;
|
||||
if (importSource === ImportDataSourceEnum.externalFile) return ExternalFileCollection;
|
||||
if (importSource === ImportDataSourceEnum.apiDataset) return APIDatasetCollection;
|
||||
}, [importSource]);
|
||||
|
||||
return ImportComponent ? (
|
||||
<Box flex={'1 0 0'} overflow={'auto'}>
|
||||
<ImportComponent />
|
||||
</Box>
|
||||
) : null;
|
||||
};
|
||||
|
||||
const Render = () => {
|
||||
return (
|
||||
<Flex
|
||||
flexDirection={'column'}
|
||||
bg={'white'}
|
||||
h={'100%'}
|
||||
px={[2, 9]}
|
||||
py={[2, 5]}
|
||||
borderRadius={'md'}
|
||||
>
|
||||
<DatasetImportContextProvider>
|
||||
<ImportDataset />
|
||||
</DatasetImportContextProvider>
|
||||
</Flex>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(Render);
|
||||
7
projects/app/src/pageComponents/dataset/detail/Import/type.d.ts
vendored
Normal file
7
projects/app/src/pageComponents/dataset/detail/Import/type.d.ts
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type';
|
||||
|
||||
export type UploadFileItemType = ImportSourceItemType & {
|
||||
file?: File;
|
||||
isUploading: boolean;
|
||||
uploadedFileRate: number;
|
||||
};
|
||||
@@ -0,0 +1,23 @@
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type';
|
||||
import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants';
|
||||
|
||||
export const getPreviewSourceReadType = (previewSource: ImportSourceItemType) => {
|
||||
if (previewSource.dbFileId) {
|
||||
return DatasetSourceReadTypeEnum.fileLocal;
|
||||
}
|
||||
if (previewSource.link) {
|
||||
return DatasetSourceReadTypeEnum.link;
|
||||
}
|
||||
if (previewSource.apiFileId) {
|
||||
return DatasetSourceReadTypeEnum.apiFile;
|
||||
}
|
||||
if (previewSource.externalFileId) {
|
||||
return DatasetSourceReadTypeEnum.externalFile;
|
||||
}
|
||||
|
||||
return DatasetSourceReadTypeEnum.fileLocal;
|
||||
};
|
||||
|
||||
export default function Dom() {
|
||||
return <></>;
|
||||
}
|
||||
Reference in New Issue
Block a user