V4.8.20 feature (#3686)
* Aiproxy (#3649) * model config * feat: model config ui * perf: rename variable * feat: custom request url * perf: model buffer * perf: init model * feat: json model config * auto login * fix: ts * update packages * package * fix: dockerfile * feat: usage filter & export & dashbord (#3538) * feat: usage filter & export & dashbord * adjust ui * fix tmb scroll * fix code & selecte all * merge * perf: usages list；perf: move components (#3654) * perf: usages list * team sub plan load * perf: usage dashboard code * perf: dashboard ui * perf: move components * add default model config (#3653) * 4.8.20 test (#3656) * provider * perf: model config * model perf (#3657) * fix: model * dataset quote * perf: model config * model tag * doubao model config * perf: config model * feat: model test * fix: POST 500 error on dingtalk bot (#3655) * feat: default model (#3662) * move model config * feat: default model * fix: false triggerd org selection (#3661) * export usage csv i18n (#3660) * export usage csv i18n * fix build * feat: markdown extension (#3663) * feat: markdown extension * media cros * rerank test * default price * perf: default model * fix: cannot custom provider * fix: default model select * update bg * perf: default model selector * fix: usage export * i18n * fix: rerank * update init extension * perf: ip limit check * doubao model order * web default modle * perf: tts selector * perf: tts error * qrcode package * reload buffer (#3665) * reload buffer * reload buffer * tts selector * fix: err tip (#3666) * fix: err tip * perf: training queue * doc * fix interactive edge (#3659) * fix interactive edge * fix * comment * add gemini model * fix: chat model select * perf: supplement assistant empty response (#3669) * perf: supplement assistant empty response * check array * perf: max_token count;feat: support resoner output;fix: member scroll (#3681) * perf: supplement assistant empty response * check array * perf: max_token count * feat: support resoner output * member scroll * update provider order * i18n * fix: stream response (#3682) * perf: supplement assistant empty response * check array * fix: stream response * fix: model config cannot set to null * fix: reasoning response (#3684) * perf: supplement assistant empty response * check array * fix: reasoning response * fix: reasoning response * doc (#3685) * perf: supplement assistant empty response * check array * doc * lock * animation * update doc * update compose * doc * doc --------- Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
This commit is contained in:
Archer
@@ -0,0 +1,283 @@
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
import React, { useCallback, useMemo, useState } from 'react';
|
||||
import dynamic from 'next/dynamic';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { Box, Button, Checkbox, Flex } from '@chakra-ui/react';
|
||||
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
import { getApiDatasetFileList, getApiDatasetFileListExistId } from '@/web/core/dataset/api';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
|
||||
import FolderPath from '@/components/common/folder/Path';
|
||||
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
|
||||
import MyBox from '@fastgpt/web/components/common/MyBox';
|
||||
import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
|
||||
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
|
||||
import { useMount } from 'ahooks';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const APIDatasetCollection = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <CustomAPIFileInput />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks={true} />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(APIDatasetCollection);
|
||||
|
||||
const CustomAPIFileInput = () => {
|
||||
const { t } = useTranslation();
|
||||
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
|
||||
const goToNext = useContextSelector(DatasetImportContext, (v) => v.goToNext);
|
||||
|
||||
const sources = useContextSelector(DatasetImportContext, (v) => v.sources);
|
||||
const setSources = useContextSelector(DatasetImportContext, (v) => v.setSources);
|
||||
|
||||
const [selectFiles, setSelectFiles] = useState<APIFileItem[]>([]);
|
||||
const [parent, setParent] = useState<ParentTreePathItemType>({
|
||||
parentId: '',
|
||||
parentName: ''
|
||||
});
|
||||
const [paths, setPaths] = useState<ParentTreePathItemType[]>([]);
|
||||
|
||||
const [searchKey, setSearchKey] = useState('');
|
||||
|
||||
const { data: fileList = [], loading } = useRequest2(
|
||||
async () => {
|
||||
return getApiDatasetFileList({
|
||||
datasetId: datasetDetail._id,
|
||||
parentId: parent?.parentId,
|
||||
searchKey: searchKey
|
||||
});
|
||||
},
|
||||
{
|
||||
refreshDeps: [datasetDetail._id, datasetDetail.apiServer, parent, searchKey],
|
||||
throttleWait: 500,
|
||||
manual: false
|
||||
}
|
||||
);
|
||||
|
||||
const { data: existIdList = [] } = useRequest2(
|
||||
() => getApiDatasetFileListExistId({ datasetId: datasetDetail._id }),
|
||||
{
|
||||
manual: false
|
||||
}
|
||||
);
|
||||
|
||||
// Init selected files
|
||||
useMount(() => {
|
||||
setSelectFiles(sources.map((item) => item.apiFile).filter(Boolean) as APIFileItem[]);
|
||||
});
|
||||
|
||||
const { runAsync: onclickNext, loading: onNextLoading } = useRequest2(
|
||||
async () => {
|
||||
// Computed all selected files
|
||||
const getFilesRecursively = async (files: APIFileItem[]): Promise<APIFileItem[]> => {
|
||||
const allFiles: APIFileItem[] = [];
|
||||
|
||||
for (const file of files) {
|
||||
if (file.type === 'folder') {
|
||||
const folderFiles = await getApiDatasetFileList({
|
||||
datasetId: datasetDetail._id,
|
||||
parentId: file?.id
|
||||
});
|
||||
|
||||
const subFiles = await getFilesRecursively(folderFiles);
|
||||
allFiles.push(...subFiles);
|
||||
} else {
|
||||
allFiles.push(file);
|
||||
}
|
||||
}
|
||||
|
||||
return allFiles;
|
||||
};
|
||||
|
||||
const allFiles = await getFilesRecursively(selectFiles);
|
||||
|
||||
setSources(
|
||||
allFiles
|
||||
.filter((item) => !existIdList.includes(item.id))
|
||||
.map((item) => ({
|
||||
id: item.id,
|
||||
apiFileId: item.id,
|
||||
apiFile: item,
|
||||
createStatus: 'waiting',
|
||||
sourceName: item.name,
|
||||
icon: getSourceNameIcon({ sourceName: item.name }) as any
|
||||
}))
|
||||
);
|
||||
},
|
||||
{
|
||||
onSuccess() {
|
||||
goToNext();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
const handleItemClick = useCallback(
|
||||
(item: APIFileItem) => {
|
||||
if (item.hasChild) {
|
||||
setPaths((state) => [...state, { parentId: item.id, parentName: item.name }]);
|
||||
return setParent({
|
||||
parentId: item.id,
|
||||
parentName: item.name
|
||||
});
|
||||
}
|
||||
|
||||
const isCurrentlySelected = selectFiles.some((file) => file.id === item.id);
|
||||
if (isCurrentlySelected) {
|
||||
setSelectFiles((state) => state.filter((file) => file.id !== item.id));
|
||||
} else {
|
||||
setSelectFiles((state) => [...state, item]);
|
||||
}
|
||||
},
|
||||
[selectFiles]
|
||||
);
|
||||
|
||||
const handleSelectAll = useCallback(() => {
|
||||
const isAllSelected = fileList.length === selectFiles.length;
|
||||
|
||||
if (isAllSelected) {
|
||||
setSelectFiles([]);
|
||||
} else {
|
||||
setSelectFiles(fileList);
|
||||
}
|
||||
}, [fileList, selectFiles]);
|
||||
|
||||
return (
|
||||
<MyBox isLoading={loading} position="relative" h="full">
|
||||
<Flex flexDirection={'column'} h="full">
|
||||
<Flex justifyContent={'space-between'}>
|
||||
<FolderPath
|
||||
paths={paths}
|
||||
onClick={(parentId) => {
|
||||
const index = paths.findIndex((item) => item.parentId === parentId);
|
||||
|
||||
setParent(paths[index]);
|
||||
setPaths(paths.slice(0, index + 1));
|
||||
}}
|
||||
/>
|
||||
{datasetDetail.apiServer && (
|
||||
<Box w={'240px'}>
|
||||
<SearchInput
|
||||
value={searchKey}
|
||||
onChange={(e) => setSearchKey(e.target.value)}
|
||||
placeholder={t('common:core.workflow.template.Search')}
|
||||
/>
|
||||
</Box>
|
||||
)}
|
||||
</Flex>
|
||||
<Box flex={1} overflowY="auto" mb={16}>
|
||||
<Box ml={2} mt={3}>
|
||||
<Flex
|
||||
alignItems={'center'}
|
||||
py={3}
|
||||
cursor={'pointer'}
|
||||
bg={'myGray.50'}
|
||||
pl={7}
|
||||
rounded={'8px'}
|
||||
fontSize={'sm'}
|
||||
fontWeight={'medium'}
|
||||
color={'myGray.900'}
|
||||
onClick={(e) => {
|
||||
if (!(e.target as HTMLElement).closest('.checkbox')) {
|
||||
handleSelectAll();
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Checkbox
|
||||
className="checkbox"
|
||||
mr={2}
|
||||
isChecked={fileList.length === selectFiles.length}
|
||||
onChange={handleSelectAll}
|
||||
/>
|
||||
{t('common:Select_all')}
|
||||
</Flex>
|
||||
{fileList.map((item) => {
|
||||
const isFolder = item.type === 'folder';
|
||||
const isExists = existIdList.includes(item.id);
|
||||
const isChecked = isExists || selectFiles.some((file) => file.id === item.id);
|
||||
|
||||
return (
|
||||
<Flex
|
||||
key={item.id}
|
||||
py={3}
|
||||
_hover={{ bg: 'primary.50' }}
|
||||
pl={7}
|
||||
cursor={'pointer'}
|
||||
onClick={(e) => {
|
||||
if (isExists) return;
|
||||
if (!(e.target as HTMLElement).closest('.checkbox')) {
|
||||
handleItemClick(item);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Checkbox
|
||||
className="checkbox"
|
||||
mr={2.5}
|
||||
isChecked={isChecked}
|
||||
isDisabled={isExists}
|
||||
onChange={(e) => {
|
||||
e.stopPropagation();
|
||||
if (isExists) return;
|
||||
if (isChecked) {
|
||||
setSelectFiles((state) => state.filter((file) => file.id !== item.id));
|
||||
} else {
|
||||
setSelectFiles((state) => [...state, item]);
|
||||
}
|
||||
}}
|
||||
/>
|
||||
<MyIcon
|
||||
name={
|
||||
!isFolder
|
||||
? (getSourceNameIcon({ sourceName: item.name }) as any)
|
||||
: 'common/folderFill'
|
||||
}
|
||||
w={'18px'}
|
||||
mr={1.5}
|
||||
/>
|
||||
<Box fontSize={'sm'} fontWeight={'medium'} color={'myGray.900'}>
|
||||
{item.name}
|
||||
</Box>
|
||||
{item.hasChild && <MyIcon name="core/chat/chevronRight" w={'18px'} ml={2} />}
|
||||
</Flex>
|
||||
);
|
||||
})}
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
<Box
|
||||
position="absolute"
|
||||
display={'flex'}
|
||||
justifyContent={'end'}
|
||||
bottom={0}
|
||||
left={0}
|
||||
right={0}
|
||||
p={4}
|
||||
>
|
||||
<Button
|
||||
isDisabled={selectFiles.length === 0}
|
||||
isLoading={onNextLoading}
|
||||
onClick={onclickNext}
|
||||
>
|
||||
{selectFiles.length > 0
|
||||
? `${t('common:core.dataset.import.Total files', { total: selectFiles.length })} | `
|
||||
: ''}
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Box>
|
||||
</Flex>
|
||||
</MyBox>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,186 @@
|
||||
import React, { useEffect } from 'react';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { useFieldArray, useForm } from 'react-hook-form';
|
||||
import {
|
||||
Box,
|
||||
Button,
|
||||
Flex,
|
||||
Table,
|
||||
Thead,
|
||||
Tbody,
|
||||
Tr,
|
||||
Th,
|
||||
Td,
|
||||
TableContainer,
|
||||
Input
|
||||
} from '@chakra-ui/react';
|
||||
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
import { getFileIcon } from '@fastgpt/global/common/file/icon';
|
||||
import { SmallAddIcon } from '@chakra-ui/icons';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const ExternalFileCollection = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <CustomLinkInput />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks={true} />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(ExternalFileCollection);
|
||||
|
||||
const CustomLinkInput = () => {
|
||||
const { t } = useTranslation();
|
||||
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const { register, reset, handleSubmit, control } = useForm<{
|
||||
list: {
|
||||
sourceName: string;
|
||||
externalFileUrl: string;
|
||||
externalFileId: string;
|
||||
}[];
|
||||
}>({
|
||||
defaultValues: {
|
||||
list: [
|
||||
{
|
||||
sourceName: '',
|
||||
externalFileUrl: '',
|
||||
externalFileId: ''
|
||||
}
|
||||
]
|
||||
}
|
||||
});
|
||||
|
||||
const {
|
||||
fields: list,
|
||||
append,
|
||||
remove,
|
||||
update
|
||||
} = useFieldArray({
|
||||
control,
|
||||
name: 'list'
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (sources.length > 0) {
|
||||
reset({
|
||||
list: sources.map((item) => ({
|
||||
sourceName: item.sourceName,
|
||||
externalFileUrl: item.externalFileUrl || '',
|
||||
externalFileId: item.externalFileId || ''
|
||||
}))
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<Box>
|
||||
<TableContainer>
|
||||
<Table bg={'white'}>
|
||||
<Thead>
|
||||
<Tr bg={'myGray.50'}>
|
||||
<Th>{t('dataset:external_url')}</Th>
|
||||
<Th>{t('dataset:external_id')}</Th>
|
||||
<Th>{t('dataset:filename')}</Th>
|
||||
<Th></Th>
|
||||
</Tr>
|
||||
</Thead>
|
||||
<Tbody>
|
||||
{list.map((item, index) => (
|
||||
<Tr key={item.id}>
|
||||
<Td>
|
||||
<Input
|
||||
{...register(`list.${index}.externalFileUrl`, {
|
||||
required: index !== list.length - 1,
|
||||
onBlur(e) {
|
||||
const val = (e.target.value || '') as string;
|
||||
if (val.includes('.') && !list[index]?.sourceName) {
|
||||
const sourceName = val.split('/').pop() || '';
|
||||
update(index, {
|
||||
...list[index],
|
||||
externalFileUrl: val,
|
||||
sourceName: decodeURIComponent(sourceName)
|
||||
});
|
||||
}
|
||||
if (val && index === list.length - 1) {
|
||||
append({
|
||||
sourceName: '',
|
||||
externalFileUrl: '',
|
||||
externalFileId: ''
|
||||
});
|
||||
}
|
||||
}
|
||||
})}
|
||||
/>
|
||||
</Td>
|
||||
<Td>
|
||||
<Input {...register(`list.${index}.externalFileId`)} />
|
||||
</Td>
|
||||
<Td>
|
||||
<Input {...register(`list.${index}.sourceName`)} />
|
||||
</Td>
|
||||
<Td>
|
||||
<MyIcon
|
||||
name={'delete'}
|
||||
w={'16px'}
|
||||
cursor={'pointer'}
|
||||
_hover={{ color: 'red.600' }}
|
||||
onClick={() => remove(index)}
|
||||
/>
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
</Tbody>
|
||||
</Table>
|
||||
</TableContainer>
|
||||
<Flex mt={5} justifyContent={'space-between'}>
|
||||
<Button
|
||||
variant={'whitePrimary'}
|
||||
leftIcon={<SmallAddIcon />}
|
||||
onClick={() => {
|
||||
append({
|
||||
sourceName: '',
|
||||
externalFileUrl: '',
|
||||
externalFileId: ''
|
||||
});
|
||||
}}
|
||||
>
|
||||
{t('common:add_new')}
|
||||
</Button>
|
||||
<Button
|
||||
isDisabled={list.filter((item) => !!item.externalFileUrl).length === 0}
|
||||
onClick={handleSubmit((data) => {
|
||||
setSources(
|
||||
data.list
|
||||
.filter((item) => !!item.externalFileUrl)
|
||||
.map((item) => ({
|
||||
id: getNanoid(32),
|
||||
createStatus: 'waiting',
|
||||
sourceName: item.sourceName || item.externalFileUrl,
|
||||
icon: getFileIcon(item.externalFileUrl),
|
||||
externalFileId: item.externalFileId,
|
||||
externalFileUrl: item.externalFileUrl
|
||||
}))
|
||||
);
|
||||
|
||||
goToNext();
|
||||
})}
|
||||
>
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,106 @@
|
||||
import React, { useCallback, useEffect } from 'react';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { Box, Button, Flex, Input, Textarea } from '@chakra-ui/react';
|
||||
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const CustomTet = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <CustomTextInput />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(CustomTet);
|
||||
|
||||
const CustomTextInput = () => {
|
||||
const { t } = useTranslation();
|
||||
const { sources, goToNext, setSources } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const { register, reset, handleSubmit } = useForm({
|
||||
defaultValues: {
|
||||
name: '',
|
||||
value: ''
|
||||
}
|
||||
});
|
||||
|
||||
const onSubmit = useCallback(
|
||||
(data: { name: string; value: string }) => {
|
||||
const fileId = getNanoid(32);
|
||||
|
||||
setSources([
|
||||
{
|
||||
id: fileId,
|
||||
createStatus: 'waiting',
|
||||
rawText: data.value,
|
||||
sourceName: data.name,
|
||||
icon: 'file/fill/manual'
|
||||
}
|
||||
]);
|
||||
goToNext();
|
||||
},
|
||||
[goToNext, setSources]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const source = sources[0];
|
||||
if (source) {
|
||||
reset({
|
||||
name: source.sourceName,
|
||||
value: source.rawText
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<Box maxW={['100%', '800px']}>
|
||||
<Box display={['block', 'flex']} alignItems={'center'}>
|
||||
<Box flex={'0 0 120px'} fontSize={'sm'}>
|
||||
{t('common:core.dataset.collection.Collection name')}
|
||||
</Box>
|
||||
<Input
|
||||
flex={'1 0 0'}
|
||||
maxW={['100%', '350px']}
|
||||
{...register('name', {
|
||||
required: true
|
||||
})}
|
||||
placeholder={t('common:core.dataset.collection.Collection name')}
|
||||
bg={'myGray.50'}
|
||||
/>
|
||||
</Box>
|
||||
<Box display={['block', 'flex']} alignItems={'flex-start'} mt={5}>
|
||||
<Box flex={'0 0 120px'} fontSize={'sm'}>
|
||||
{t('common:core.dataset.collection.Collection raw text')}
|
||||
</Box>
|
||||
<Textarea
|
||||
flex={'1 0 0'}
|
||||
w={'100%'}
|
||||
rows={15}
|
||||
placeholder={t('common:core.dataset.collection.Collection raw text')}
|
||||
{...register('value', {
|
||||
required: true
|
||||
})}
|
||||
bg={'myGray.50'}
|
||||
/>
|
||||
</Box>
|
||||
<Flex mt={5} justifyContent={'flex-end'}>
|
||||
<Button onClick={handleSubmit((data) => onSubmit(data))}>
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,153 @@
|
||||
import React, { useEffect } from 'react';
|
||||
import dynamic from 'next/dynamic';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { Box, Button, Flex, Input, Link, Textarea } from '@chakra-ui/react';
|
||||
import { getNanoid } from '@fastgpt/global/common/string/tools';
|
||||
import MyIcon from '@fastgpt/web/components/common/Icon';
|
||||
import { LinkCollectionIcon } from '@fastgpt/global/core/dataset/constants';
|
||||
import { useSystemStore } from '@/web/common/system/useSystemStore';
|
||||
import { getDocPath } from '@/web/common/system/doc';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const LinkCollection = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <CustomLinkImport />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(LinkCollection);
|
||||
|
||||
const CustomLinkImport = () => {
|
||||
const { t } = useTranslation();
|
||||
const { feConfigs } = useSystemStore();
|
||||
const { goToNext, sources, setSources, processParamsForm } = useContextSelector(
|
||||
DatasetImportContext,
|
||||
(v) => v
|
||||
);
|
||||
const { register, reset, handleSubmit, watch } = useForm({
|
||||
defaultValues: {
|
||||
link: ''
|
||||
}
|
||||
});
|
||||
|
||||
const link = watch('link');
|
||||
const linkList = link.split('\n').filter((item) => item);
|
||||
|
||||
useEffect(() => {
|
||||
reset({
|
||||
link: sources
|
||||
.map((item) => item.link)
|
||||
.filter((item) => item)
|
||||
.join('\n')
|
||||
});
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<Box maxW={['100%', '800px']}>
|
||||
<Box display={['block', 'flex']} alignItems={'flex-start'} mt={1}>
|
||||
<Box flex={'0 0 100px'} fontSize={'sm'}>
|
||||
{t('common:core.dataset.import.Link name')}
|
||||
</Box>
|
||||
<Textarea
|
||||
flex={'1 0 0'}
|
||||
w={'100%'}
|
||||
rows={10}
|
||||
placeholder={t('common:core.dataset.import.Link name placeholder')}
|
||||
bg={'myGray.50'}
|
||||
overflowX={'auto'}
|
||||
whiteSpace={'nowrap'}
|
||||
{...register('link', {
|
||||
required: true
|
||||
})}
|
||||
/>
|
||||
</Box>
|
||||
<Box display={['block', 'flex']} alignItems={'center'} mt={4}>
|
||||
<Box flex={'0 0 100px'} fontSize={'sm'}>
|
||||
{t('common:core.dataset.website.Selector')}
|
||||
<Box color={'myGray.500'} fontSize={'sm'}>
|
||||
{feConfigs?.docUrl && (
|
||||
<Link
|
||||
href={getDocPath('/docs/guide/knowledge_base/websync/#选择器如何使用')}
|
||||
target="_blank"
|
||||
>
|
||||
{t('common:core.dataset.website.Selector Course')}
|
||||
</Link>
|
||||
)}
|
||||
</Box>
|
||||
</Box>
|
||||
<Input
|
||||
flex={'1 0 0'}
|
||||
maxW={['100%', '350px']}
|
||||
{...processParamsForm.register('webSelector')}
|
||||
placeholder={'body .content #document'}
|
||||
bg={'myGray.50'}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
<Flex my={4} flexWrap={'wrap'} gap={4} alignItems={'center'} pl={[0, '100px']}>
|
||||
{linkList.map((item, i) => (
|
||||
<Flex
|
||||
key={`${item}-${i}`}
|
||||
alignItems={'center'}
|
||||
px={4}
|
||||
py={2}
|
||||
borderRadius={'md'}
|
||||
bg={'myGray.100'}
|
||||
>
|
||||
<MyIcon name={LinkCollectionIcon} w={'16px'} />
|
||||
<Box ml={1} mr={3} wordBreak={'break-all'}>
|
||||
{item}
|
||||
</Box>
|
||||
<MyIcon
|
||||
name={'common/closeLight'}
|
||||
w={'14px'}
|
||||
color={'myGray.500'}
|
||||
cursor={'pointer'}
|
||||
onClick={() => {
|
||||
const newLinkList = linkList.filter((link, index) => index !== i);
|
||||
reset({
|
||||
link: newLinkList.join('\n')
|
||||
});
|
||||
}}
|
||||
/>
|
||||
</Flex>
|
||||
))}
|
||||
</Flex>
|
||||
<Flex mt={5} justifyContent={'flex-end'}>
|
||||
<Button
|
||||
onClick={handleSubmit((data) => {
|
||||
const newLinkList = data.link.split('\n').filter((item) => item);
|
||||
|
||||
setSources(
|
||||
newLinkList.map((link) => ({
|
||||
id: getNanoid(32),
|
||||
createStatus: 'waiting',
|
||||
link,
|
||||
sourceName: link,
|
||||
icon: LinkCollectionIcon
|
||||
}))
|
||||
);
|
||||
|
||||
goToNext();
|
||||
})}
|
||||
>
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Flex>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,79 @@
|
||||
import React, { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
|
||||
import { Box, Button } from '@chakra-ui/react';
|
||||
import FileSelector from '../components/FileSelector';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
import Loading from '@fastgpt/web/components/common/MyLoading';
|
||||
import { RenderUploadFiles } from '../components/RenderFiles';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
|
||||
loading: () => <Loading fixed={false} />
|
||||
});
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const fileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx';
|
||||
|
||||
const FileLocal = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <SelectFile />}
|
||||
{activeStep === 1 && <DataProcess showPreviewChunks />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(FileLocal);
|
||||
|
||||
const SelectFile = React.memo(function SelectFile() {
|
||||
const { t } = useTranslation();
|
||||
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>(
|
||||
sources.map((source) => ({
|
||||
isUploading: false,
|
||||
...source
|
||||
}))
|
||||
);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
|
||||
|
||||
useEffect(() => {
|
||||
setSources(successFiles);
|
||||
}, [setSources, successFiles]);
|
||||
|
||||
const onclickNext = useCallback(() => {
|
||||
// filter uploaded files
|
||||
setSelectFiles((state) => state.filter((item) => item.dbFileId));
|
||||
goToNext();
|
||||
}, [goToNext]);
|
||||
|
||||
return (
|
||||
<Box>
|
||||
<FileSelector
|
||||
fileType={fileType}
|
||||
selectFiles={selectFiles}
|
||||
setSelectFiles={setSelectFiles}
|
||||
onStartSelect={() => setUploading(true)}
|
||||
onFinishSelect={() => setUploading(false)}
|
||||
/>
|
||||
|
||||
{/* render files */}
|
||||
<RenderUploadFiles files={selectFiles} setFiles={setSelectFiles} showPreviewContent />
|
||||
|
||||
<Box textAlign={'right'} mt={5}>
|
||||
<Button isDisabled={successFiles.length === 0 || uploading} onClick={onclickNext}>
|
||||
{selectFiles.length > 0
|
||||
? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
|
||||
: ''}
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
import React from 'react';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
import DataProcess from '../commonProgress/DataProcess';
|
||||
import { useRouter } from 'next/router';
|
||||
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
|
||||
import { getDatasetCollectionById } from '@/web/core/dataset/api';
|
||||
import MyBox from '@fastgpt/web/components/common/MyBox';
|
||||
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
|
||||
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
|
||||
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const ReTraining = () => {
|
||||
const router = useRouter();
|
||||
|
||||
const { collectionId = '' } = router.query as {
|
||||
collectionId: string;
|
||||
};
|
||||
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
const setSources = useContextSelector(DatasetImportContext, (v) => v.setSources);
|
||||
const processParamsForm = useContextSelector(DatasetImportContext, (v) => v.processParamsForm);
|
||||
|
||||
const { loading } = useRequest2(() => getDatasetCollectionById(collectionId), {
|
||||
refreshDeps: [collectionId],
|
||||
manual: false,
|
||||
onSuccess: (collection) => {
|
||||
setSources([
|
||||
{
|
||||
dbFileId: collection.fileId,
|
||||
link: collection.rawLink,
|
||||
apiFileId: collection.apiFileId,
|
||||
|
||||
createStatus: 'waiting',
|
||||
icon: getCollectionIcon(collection.type, collection.name),
|
||||
id: collection._id,
|
||||
isUploading: false,
|
||||
sourceName: collection.name,
|
||||
uploadedFileRate: 100
|
||||
}
|
||||
]);
|
||||
processParamsForm.reset({
|
||||
mode: collection.trainingType,
|
||||
way: ImportProcessWayEnum.auto,
|
||||
embeddingChunkSize: collection.chunkSize,
|
||||
qaChunkSize: collection.chunkSize,
|
||||
customSplitChar: collection.chunkSplitter,
|
||||
qaPrompt: collection.qaPrompt,
|
||||
webSelector: collection.metadata?.webSelector
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return (
|
||||
<MyBox isLoading={loading} h={'100%'} overflow={'auto'}>
|
||||
{activeStep === 0 && <DataProcess showPreviewChunks={true} />}
|
||||
{activeStep === 1 && <Upload />}
|
||||
</MyBox>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(ReTraining);
|
||||
@@ -0,0 +1,101 @@
|
||||
import React, { useEffect, useMemo, useState } from 'react';
|
||||
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
|
||||
import { Box, Button } from '@chakra-ui/react';
|
||||
import FileSelector from '../components/FileSelector';
|
||||
import { useTranslation } from 'next-i18next';
|
||||
|
||||
import dynamic from 'next/dynamic';
|
||||
import { fileDownload } from '@/web/common/file/utils';
|
||||
import { RenderUploadFiles } from '../components/RenderFiles';
|
||||
import { useContextSelector } from 'use-context-selector';
|
||||
import { DatasetImportContext } from '../Context';
|
||||
|
||||
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
|
||||
const Upload = dynamic(() => import('../commonProgress/Upload'));
|
||||
|
||||
const fileType = '.csv';
|
||||
|
||||
const FileLocal = () => {
|
||||
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
|
||||
|
||||
return (
|
||||
<>
|
||||
{activeStep === 0 && <SelectFile />}
|
||||
{activeStep === 1 && <PreviewData showPreviewChunks />}
|
||||
{activeStep === 2 && <Upload />}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default React.memo(FileLocal);
|
||||
|
||||
const csvTemplate = `index,content
|
||||
"第一列内容","第二列内容"
|
||||
"必填列","可选列。CSV 中请注意内容不能包含双引号，双引号是列分割符号"
|
||||
"只会将第一和第二列内容导入，其余列会被忽略",""
|
||||
"结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。",""
|
||||
"AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`;
|
||||
|
||||
const SelectFile = React.memo(function SelectFile() {
|
||||
const { t } = useTranslation();
|
||||
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
|
||||
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>(
|
||||
sources.map((source) => ({
|
||||
isUploading: false,
|
||||
...source
|
||||
}))
|
||||
);
|
||||
const [uploading, setUploading] = useState(false);
|
||||
|
||||
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
|
||||
|
||||
useEffect(() => {
|
||||
setSources(successFiles);
|
||||
}, [successFiles]);
|
||||
|
||||
return (
|
||||
<Box>
|
||||
<FileSelector
|
||||
fileType={fileType}
|
||||
selectFiles={selectFiles}
|
||||
setSelectFiles={setSelectFiles}
|
||||
onStartSelect={() => setUploading(true)}
|
||||
onFinishSelect={() => setUploading(false)}
|
||||
/>
|
||||
|
||||
<Box
|
||||
mt={4}
|
||||
color={'primary.600'}
|
||||
textDecoration={'underline'}
|
||||
cursor={'pointer'}
|
||||
onClick={() =>
|
||||
fileDownload({
|
||||
text: csvTemplate,
|
||||
type: 'text/csv;charset=utf-8',
|
||||
filename: 'template.csv'
|
||||
})
|
||||
}
|
||||
>
|
||||
{t('common:core.dataset.import.Down load csv template')}
|
||||
</Box>
|
||||
|
||||
{/* render files */}
|
||||
<RenderUploadFiles files={selectFiles} setFiles={setSelectFiles} />
|
||||
|
||||
<Box textAlign={'right'} mt={5}>
|
||||
<Button
|
||||
isDisabled={successFiles.length === 0 || uploading}
|
||||
onClick={() => {
|
||||
setSelectFiles((state) => state.filter((item) => !item.errorMsg));
|
||||
goToNext();
|
||||
}}
|
||||
>
|
||||
{selectFiles.length > 0
|
||||
? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
|
||||
: ''}
|
||||
{t('common:common.Next Step')}
|
||||
</Button>
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user