gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V4.8.20 feature (#3686)

Browse Source 
* Aiproxy (#3649)

* model config

* feat: model config ui

* perf: rename variable

* feat: custom request url

* perf: model buffer

* perf: init model

* feat: json model config

* auto login

* fix: ts

* update packages

* package

* fix: dockerfile

* feat: usage filter & export & dashbord (#3538)

* feat: usage filter & export & dashbord

* adjust ui

* fix tmb scroll

* fix code & selecte all

* merge

* perf: usages list；perf: move components (#3654)

* perf: usages list

* team sub plan load

* perf: usage dashboard code

* perf: dashboard ui

* perf: move components

* add default model config (#3653)

* 4.8.20 test (#3656)

* provider

* perf: model config

* model perf (#3657)

* fix: model

* dataset quote

* perf: model config

* model tag

* doubao model config

* perf: config model

* feat: model test

* fix: POST 500 error on dingtalk bot (#3655)

* feat: default model (#3662)

* move model config

* feat: default model

* fix: false triggerd org selection (#3661)

* export usage csv i18n (#3660)

* export usage csv i18n

* fix build

* feat: markdown extension (#3663)

* feat: markdown extension

* media cros

* rerank test

* default price

* perf: default model

* fix: cannot custom provider

* fix: default model select

* update bg

* perf: default model selector

* fix: usage export

* i18n

* fix: rerank

* update init extension

* perf: ip limit check

* doubao model order

* web default modle

* perf: tts selector

* perf: tts error

* qrcode package

* reload buffer (#3665)

* reload buffer

* reload buffer

* tts selector

* fix: err tip (#3666)

* fix: err tip

* perf: training queue

* doc

* fix interactive edge (#3659)

* fix interactive edge

* fix

* comment

* add gemini model

* fix: chat model select

* perf: supplement assistant empty response (#3669)

* perf: supplement assistant empty response

* check array

* perf: max_token count;feat: support resoner output;fix: member scroll (#3681)

* perf: supplement assistant empty response

* check array

* perf: max_token count

* feat: support resoner output

* member scroll

* update provider order

* i18n

* fix: stream response (#3682)

* perf: supplement assistant empty response

* check array

* fix: stream response

* fix: model config cannot set to null

* fix: reasoning response (#3684)

* perf: supplement assistant empty response

* check array

* fix: reasoning response

* fix: reasoning response

* doc (#3685)

* perf: supplement assistant empty response

* check array

* doc

* lock

* animation

* update doc

* update compose

* doc

* doc

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
This commit is contained in:
Archer
2025-02-05 00:10:47 +08:00
committed by GitHub
parent c393002f1d
commit db2c0a0bdb
496 changed files with 9031 additions and 4726 deletions
Download Patch File Download Diff File Expand all files Collapse all files

166
projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx Normal file
View File

@@ -0,0 +1,166 @@
import React from 'react';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { Flex, Input } from '@chakra-ui/react';
import { UseFormReturn } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import type {
APIFileServer,
FeishuServer,
YuqueServer
} from '@fastgpt/global/core/dataset/apiDataset';
const ApiDatasetForm = ({
type,
form
}: {
type: `${DatasetTypeEnum}`;
form: UseFormReturn<
{
apiServer?: APIFileServer;
feishuServer?: FeishuServer;
yuqueServer?: YuqueServer;
},
any
>;
}) => {
const { t } = useTranslation();
const { register } = form;
return (
<>
{type === DatasetTypeEnum.apiDataset && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
{t('dataset:api_url')}
</Flex>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:api_url')}
maxLength={200}
{...register('apiServer.baseUrl', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
Authorization
</Flex>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
maxLength={200}
{...register('apiServer.authorization')}
/>
</Flex>
</>
)}
{type === DatasetTypeEnum.feishu && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
App ID
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'App ID'}
maxLength={200}
{...register('feishuServer.appId', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
App Secret
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'App Secret'}
maxLength={200}
{...register('feishuServer.appSecret', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
Folder Token
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'Folder Token'}
maxLength={200}
{...register('feishuServer.folderToken', { required: true })}
/>
</Flex>
</>
)}
{type === DatasetTypeEnum.yuque && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
User ID
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'User ID'}
maxLength={200}
{...register('yuqueServer.userId', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
Token
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'Token'}
maxLength={200}
{...register('yuqueServer.token', { required: true })}
/>
</Flex>
</>
)}
</>
);
};
export default ApiDatasetForm;

76
projects/app/src/pageComponents/dataset/EditFolderModal.tsx Normal file
View File

@@ -0,0 +1,76 @@
import React, { useMemo, useRef, useState } from 'react';
import { ModalFooter, ModalBody, Input, Button } from '@chakra-ui/react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useTranslation } from 'next-i18next';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
const EditFolderModal = ({
onClose,
editCallback,
isEdit = false,
name
}: {
onClose: () => void;
editCallback: (name: string) => Promise<any>;
isEdit: boolean;
name?: string;
}) => {
const { t } = useTranslation();
const inputRef = useRef<HTMLInputElement>(null);
const typeMap = useMemo(
() =>
isEdit
? {
title: t('common:dataset.Edit Folder')
}
: {
title: t('common:dataset.Create Folder')
},
[isEdit, t]
);
const { mutate: onSave, isLoading } = useRequest({
mutationFn: () => {
const val = inputRef.current?.value;
if (!val) return Promise.resolve('');
return editCallback(val);
},
onSuccess: () => {
onClose();
}
});
return (
<MyModal isOpen onClose={onClose} iconSrc="common/folderFill" title={typeMap.title}>
<ModalBody>
<Input
ref={inputRef}
defaultValue={name}
placeholder={t('common:dataset.Folder Name') || ''}
autoFocus
maxLength={20}
/>
</ModalBody>
<ModalFooter>
<Button isLoading={isLoading} onClick={onSave}>
{t('common:common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};
export default EditFolderModal;
export const useEditFolder = () => {
const [editFolderData, setEditFolderData] = useState<{
id?: string;
name?: string;
}>();
return {
editFolderData,
setEditFolderData
};
};

55
projects/app/src/pageComponents/dataset/MemberManager.tsx Normal file
View File

@@ -0,0 +1,55 @@
import { Box, Flex } from '@chakra-ui/react';
import React from 'react';
import CollaboratorContextProvider, {
MemberManagerInputPropsType
} from '@/components/support/permission/MemberManager/context';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
function MemberManager({ managePer }: { managePer: MemberManagerInputPropsType }) {
const { t } = useTranslation();
return (
<Box>
<CollaboratorContextProvider {...managePer}>
{({ MemberListCard, onOpenManageModal, onOpenAddMember }) => {
return (
<>
<Flex alignItems="center" flexDirection="row" justifyContent="space-between" w="full">
<Box color={'myGray.900'} fontSize={'mini'} fontWeight={'bold'}>
{t('common:permission.Collaborator')}
</Box>
<Flex gap={2}>
<Box>
<MyIcon
onClick={onOpenManageModal}
name="common/setting"
w={'1rem'}
h={'1rem'}
color={'myGray.600'}
cursor={'pointer'}
_hover={{ color: 'primary.500' }}
/>
</Box>
<Box>
<MyIcon
cursor={'pointer'}
onClick={onOpenAddMember}
name="common/addUser"
_hover={{ color: 'primary.500' }}
w={'1rem'}
h={'1rem'}
color={'myGray.600'}
/>
</Box>
</Flex>
</Flex>
<MemberListCard mt={2} p={1.5} bg="myGray.100" borderRadius="md" />
</>
);
}}
</CollaboratorContextProvider>
</Box>
);
}
export default MemberManager;

164
projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx Normal file
View File

@@ -0,0 +1,164 @@
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { Dispatch, ReactNode, SetStateAction, useEffect, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { createContext, useContextSelector } from 'use-context-selector';
import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import { useDisclosure } from '@chakra-ui/react';
import { checkTeamWebSyncLimit } from '@/web/support/user/team/api';
import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api';
import { getDatasetCollections, postWebsiteSync } from '@/web/core/dataset/api';
import dynamic from 'next/dynamic';
import { usePagination } from '@fastgpt/web/hooks/usePagination';
import { DatasetCollectionsListItemType } from '@/global/core/dataset/type';
import { useRouter } from 'next/router';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
const WebSiteConfigModal = dynamic(() => import('./WebsiteConfig'));
type CollectionPageContextType = {
openWebSyncConfirm: () => void;
onOpenWebsiteModal: () => void;
collections: DatasetCollectionsListItemType[];
Pagination: () => JSX.Element;
total: number;
getData: (e: number) => void;
isGetting: boolean;
pageNum: number;
pageSize: number;
searchText: string;
setSearchText: Dispatch<SetStateAction<string>>;
filterTags: string[];
setFilterTags: Dispatch<SetStateAction<string[]>>;
};
export const CollectionPageContext = createContext<CollectionPageContextType>({
openWebSyncConfirm: function (): () => void {
throw new Error('Function not implemented.');
},
onOpenWebsiteModal: function (): void {
throw new Error('Function not implemented.');
},
collections: [],
Pagination: function (): JSX.Element {
throw new Error('Function not implemented.');
},
total: 0,
getData: function (e: number): void {
throw new Error('Function not implemented.');
},
isGetting: false,
pageNum: 0,
pageSize: 0,
searchText: '',
setSearchText: function (value: SetStateAction<string>): void {
throw new Error('Function not implemented.');
},
filterTags: [],
setFilterTags: function (value: SetStateAction<string[]>): void {
throw new Error('Function not implemented.');
}
});
const CollectionPageContextProvider = ({ children }: { children: ReactNode }) => {
const { t } = useTranslation();
const router = useRouter();
const { parentId = '' } = router.query as { parentId: string };
const { datasetDetail, datasetId, updateDataset } = useContextSelector(
DatasetPageContext,
(v) => v
);
// website config
const { openConfirm: openWebSyncConfirm, ConfirmModal: ConfirmWebSyncModal } = useConfirm({
content: t('dataset:start_sync_website_tip')
});
const {
isOpen: isOpenWebsiteModal,
onOpen: onOpenWebsiteModal,
onClose: onCloseWebsiteModal
} = useDisclosure();
const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({
mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => {
onCloseWebsiteModal();
await checkTeamWebSyncLimit();
await updateDataset({
id: datasetId,
websiteConfig,
status: DatasetStatusEnum.syncing
});
const billId = await postCreateTrainingUsage({
name: t('common:core.dataset.training.Website Sync'),
datasetId: datasetId
});
await postWebsiteSync({ datasetId: datasetId, billId });
return;
},
errorToast: t('common:common.Update Failed')
});
// collection list
const [searchText, setSearchText] = useState('');
const [filterTags, setFilterTags] = useState<string[]>([]);
const {
data: collections,
Pagination,
total,
getData,
isLoading: isGetting,
pageNum,
pageSize
} = usePagination(getDatasetCollections, {
pageSize: 20,
params: {
datasetId,
parentId,
searchText,
filterTags
},
// defaultRequest: false,
refreshDeps: [parentId, searchText, filterTags]
});
const contextValue: CollectionPageContextType = {
openWebSyncConfirm: openWebSyncConfirm(onUpdateDatasetWebsiteConfig),
onOpenWebsiteModal,
searchText,
setSearchText,
filterTags,
setFilterTags,
collections,
Pagination,
total,
getData,
isGetting,
pageNum,
pageSize
};
return (
<CollectionPageContext.Provider value={contextValue}>
{children}
{datasetDetail.type === DatasetTypeEnum.websiteDataset && (
<>
{isOpenWebsiteModal && (
<WebSiteConfigModal
onClose={onCloseWebsiteModal}
onSuccess={onUpdateDatasetWebsiteConfig}
defaultValue={{
url: datasetDetail?.websiteConfig?.url,
selector: datasetDetail?.websiteConfig?.selector
}}
/>
)}
<ConfirmWebSyncModal />
</>
)}
</CollectionPageContext.Provider>
);
};
export default CollectionPageContextProvider;

55
projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx Normal file
View File

@@ -0,0 +1,55 @@
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import React from 'react';
import { useTranslation } from 'next-i18next';
import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { Box, Flex } from '@chakra-ui/react';
import { useContextSelector } from 'use-context-selector';
import { CollectionPageContext } from './Context';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
const EmptyCollectionTip = () => {
const { t } = useTranslation();
const onOpenWebsiteModal = useContextSelector(CollectionPageContext, (v) => v.onOpenWebsiteModal);
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
return (
<>
{(datasetDetail.type === DatasetTypeEnum.dataset ||
datasetDetail.type === DatasetTypeEnum.externalFile) && (
<EmptyTip text={t('common:core.dataset.collection.Empty Tip')} />
)}
{datasetDetail.type === DatasetTypeEnum.websiteDataset && (
<EmptyTip
text={
<Flex>
{datasetDetail.status === DatasetStatusEnum.syncing && (
<>{t('common:core.dataset.status.syncing')}</>
)}
{datasetDetail.status === DatasetStatusEnum.active && (
<>
{!datasetDetail?.websiteConfig?.url ? (
<>
{t('common:core.dataset.collection.Website Empty Tip')}
{', '}
<Box
textDecoration={'underline'}
cursor={'pointer'}
onClick={onOpenWebsiteModal}
>
{t('common:core.dataset.collection.Click top config website')}
</Box>
</>
) : (
<>{t('common:core.dataset.website.UnValid Website Tip')}</>
)}
</>
)}
</Flex>
}
/>
)}
</>
);
};
export default EmptyCollectionTip;

453
projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx Normal file
View File

@@ -0,0 +1,453 @@
import React from 'react';
import {
Box,
Flex,
MenuButton,
Button,
Link,
useTheme,
useDisclosure,
HStack
} from '@chakra-ui/react';
import {
getDatasetCollectionPathById,
postDatasetCollection,
putDatasetCollectionById
} from '@/web/core/dataset/api';
import { useQuery } from '@tanstack/react-query';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyInput from '@/components/MyInput';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { useRouter } from 'next/router';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyMenu from '@fastgpt/web/components/common/MyMenu';
import { useEditTitle } from '@/web/common/hooks/useEditTitle';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum,
DatasetTypeEnum,
DatasetTypeMap,
DatasetStatusEnum
} from '@fastgpt/global/core/dataset/constants';
import EditFolderModal, { useEditFolder } from '../../EditFolderModal';
import { TabEnum } from '../../../../pages/dataset/detail/index';
import ParentPath from '@/components/common/ParentPaths';
import dynamic from 'next/dynamic';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { useContextSelector } from 'use-context-selector';
import { CollectionPageContext } from './Context';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import HeaderTagPopOver from './HeaderTagPopOver';
const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector'));
const Header = ({}: {}) => {
const { t } = useTranslation();
const theme = useTheme();
const { setLoading, feConfigs } = useSystemStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const router = useRouter();
const { parentId = '' } = router.query as { parentId: string };
const { isPc } = useSystem();
const { searchText, setSearchText, total, getData, pageNum, onOpenWebsiteModal } =
useContextSelector(CollectionPageContext, (v) => v);
const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
getDatasetCollectionPathById(parentId)
);
const { editFolderData, setEditFolderData } = useEditFolder();
const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } =
useEditTitle({
title: t('common:dataset.Create manual collection'),
tip: t('common:dataset.Manual collection Tip'),
canEmpty: false
});
const {
isOpen: isOpenFileSourceSelector,
onOpen: onOpenFileSourceSelector,
onClose: onCloseFileSourceSelector
} = useDisclosure();
const { mutate: onCreateCollection } = useRequest({
mutationFn: async ({
name,
type,
callback,
...props
}: {
name: string;
type: DatasetCollectionTypeEnum;
callback?: (id: string) => void;
trainingType?: TrainingModeEnum;
rawLink?: string;
chunkSize?: number;
}) => {
setLoading(true);
const id = await postDatasetCollection({
parentId,
datasetId: datasetDetail._id,
name,
type,
...props
});
callback?.(id);
return id;
},
onSuccess() {
getData(pageNum);
},
onSettled() {
setLoading(false);
},
successToast: t('common:common.Create Success'),
errorToast: t('common:common.Create Failed')
});
const isWebSite = datasetDetail?.type === DatasetTypeEnum.websiteDataset;
return (
<Box display={['block', 'flex']} alignItems={'center'} gap={2}>
<HStack flex={1}>
<Box flex={1} fontWeight={'500'} color={'myGray.900'} whiteSpace={'nowrap'}>
<ParentPath
paths={paths.map((path, i) => ({
parentId: path.parentId,
parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName
}))}
FirstPathDom={
<Flex
flexDir={'column'}
justify={'center'}
h={'100%'}
fontSize={isWebSite ? 'sm' : 'md'}
fontWeight={'500'}
color={'myGray.600'}
>
<Flex align={'center'}>
{!isWebSite && <MyIcon name="common/list" mr={2} w={'20px'} color={'black'} />}
{t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel as any)}({total})
</Flex>
{datasetDetail?.websiteConfig?.url && (
<Flex fontSize={'mini'}>
{t('common:core.dataset.website.Base Url')}:
<Link
href={datasetDetail.websiteConfig.url}
target="_blank"
mr={2}
color={'blue.700'}
>
{datasetDetail.websiteConfig.url}
</Link>
</Flex>
)}
</Flex>
}
onClick={(e) => {
router.replace({
query: {
...router.query,
parentId: e
}
});
}}
/>
</Box>
{/* search input */}
{isPc && (
<MyInput
maxW={'250px'}
flex={1}
size={'sm'}
h={'36px'}
placeholder={t('common:common.Search') || ''}
value={searchText}
leftIcon={
<MyIcon
name="common/searchLight"
position={'absolute'}
w={'16px'}
color={'myGray.500'}
/>
}
onChange={(e) => {
setSearchText(e.target.value);
}}
/>
)}
{/* Tag */}
{datasetDetail.permission.hasWritePer && feConfigs?.isPlus && <HeaderTagPopOver />}
</HStack>
{/* diff collection button */}
{datasetDetail.permission.hasWritePer && (
<Box textAlign={'end'} mt={[3, 0]}>
{datasetDetail?.type === DatasetTypeEnum.dataset && (
<MyMenu
offset={[0, 5]}
Button={
<MenuButton
_hover={{
color: 'primary.500'
}}
fontSize={['sm', 'md']}
>
<Flex
px={3.5}
py={2}
borderRadius={'sm'}
cursor={'pointer'}
bg={'primary.500'}
overflow={'hidden'}
color={'white'}
>
<Flex h={'20px'} alignItems={'center'}>
<MyIcon
name={'common/folderImport'}
mr={2}
w={'18px'}
h={'18px'}
color={'white'}
/>
</Flex>
<Box h={'20px'} fontSize={'sm'} fontWeight={'500'}>
{t('common:dataset.collections.Create And Import')}
</Box>
</Flex>
</MenuButton>
}
menuList={[
{
children: [
{
label: (
<Flex>
<MyIcon name={'common/folderFill'} w={'20px'} mr={2} />
{t('common:Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
label: (
<Flex>
<MyIcon name={'core/dataset/manualCollection'} mr={2} w={'20px'} />
{t('common:core.dataset.Manual collection')}
</Flex>
),
onClick: () => {
onOpenCreateVirtualFileModal({
defaultVal: '',
onSuccess: (name) => {
onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual });
}
});
}
},
{
label: (
<Flex>
<MyIcon name={'core/dataset/fileCollection'} mr={2} w={'20px'} />
{t('common:core.dataset.Text collection')}
</Flex>
),
onClick: onOpenFileSourceSelector
},
{
label: (
<Flex>
<MyIcon name={'core/dataset/tableCollection'} mr={2} w={'20px'} />
{t('common:core.dataset.Table collection')}
</Flex>
),
onClick: () =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.import,
source: ImportDataSourceEnum.csvTable
}
})
}
]
}
]}
/>
)}
{datasetDetail?.type === DatasetTypeEnum.websiteDataset && (
<>
{datasetDetail?.websiteConfig?.url ? (
<Flex alignItems={'center'}>
{datasetDetail.status === DatasetStatusEnum.active && (
<Button onClick={onOpenWebsiteModal}>{t('common:common.Config')}</Button>
)}
{datasetDetail.status === DatasetStatusEnum.syncing && (
<Flex
ml={3}
alignItems={'center'}
px={3}
py={1}
borderRadius="md"
border={theme.borders.base}
>
<Box
animation={'zoomStopIcon 0.5s infinite alternate'}
bg={'myGray.700'}
w="8px"
h="8px"
borderRadius={'50%'}
mt={'1px'}
></Box>
<Box ml={2} color={'myGray.600'}>
{t('common:core.dataset.status.syncing')}
</Box>
</Flex>
)}
</Flex>
) : (
<Button onClick={onOpenWebsiteModal}>
{t('common:core.dataset.Set Website Config')}
</Button>
)}
</>
)}
{datasetDetail?.type === DatasetTypeEnum.externalFile && (
<MyMenu
offset={[0, 5]}
Button={
<MenuButton
_hover={{
color: 'primary.500'
}}
fontSize={['sm', 'md']}
>
<Flex
px={3.5}
py={2}
borderRadius={'sm'}
cursor={'pointer'}
bg={'primary.500'}
overflow={'hidden'}
color={'white'}
>
<Flex h={'20px'} alignItems={'center'}>
<MyIcon
name={'common/folderImport'}
mr={2}
w={'18px'}
h={'18px'}
color={'white'}
/>
</Flex>
<Box h={'20px'} fontSize={'sm'} fontWeight={'500'}>
{t('common:dataset.collections.Create And Import')}
</Box>
</Flex>
</MenuButton>
}
menuList={[
{
children: [
{
label: (
<Flex>
<MyIcon name={'common/folderFill'} w={'20px'} mr={2} />
{t('common:Folder')}
</Flex>
),
onClick: () => setEditFolderData({})
},
{
label: (
<Flex>
<MyIcon name={'core/dataset/fileCollection'} mr={2} w={'20px'} />
{t('common:core.dataset.Text collection')}
</Flex>
),
onClick: () =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.import,
source: ImportDataSourceEnum.externalFile
}
})
}
]
}
]}
/>
)}
{/* apiDataset */}
{(datasetDetail?.type === DatasetTypeEnum.apiDataset ||
datasetDetail?.type === DatasetTypeEnum.feishu ||
datasetDetail?.type === DatasetTypeEnum.yuque) && (
<Flex
px={3.5}
py={2}
borderRadius={'sm'}
cursor={'pointer'}
bg={'primary.500'}
overflow={'hidden'}
color={'white'}
onClick={() =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.import,
source: ImportDataSourceEnum.apiDataset
}
})
}
>
<Flex h={'20px'} alignItems={'center'}>
<MyIcon name={'common/folderImport'} mr={2} w={'18px'} h={'18px'} color={'white'} />
</Flex>
<Box h={'20px'} fontSize={'sm'} fontWeight={'500'}>
{t('dataset:add_file')}
</Box>
</Flex>
)}
</Box>
)}
{/* modal */}
{!!editFolderData && (
<EditFolderModal
onClose={() => setEditFolderData(undefined)}
editCallback={async (name) => {
try {
if (editFolderData.id) {
await putDatasetCollectionById({
id: editFolderData.id,
name
});
getData(pageNum);
} else {
onCreateCollection({
name,
type: DatasetCollectionTypeEnum.folder
});
}
} catch (error) {
return Promise.reject(error);
}
}}
isEdit={!!editFolderData.id}
name={editFolderData.name}
/>
)}
<EditCreateVirtualFileModal iconSrc={'modal/manualDataset'} closeBtnText={''} />
{isOpenFileSourceSelector && <FileSourceSelector onClose={onCloseFileSourceSelector} />}
</Box>
);
};
export default Header;

211
projects/app/src/pageComponents/dataset/detail/CollectionCard/HeaderTagPopOver.tsx Normal file
View File

@@ -0,0 +1,211 @@
import { Box, Button, Checkbox, Flex, Input, useDisclosure } from '@chakra-ui/react';
import MyPopover from '@fastgpt/web/components/common/MyPopover';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useTranslation } from 'next-i18next';
import { CollectionPageContext } from './Context';
import { isEqual } from 'lodash';
import TagManageModal from './TagManageModal';
import { DatasetTagType } from '@fastgpt/global/core/dataset/type';
const HeaderTagPopOver = () => {
const { t } = useTranslation();
const {
searchDatasetTagsResult,
searchTagKey,
setSearchTagKey,
checkedDatasetTag,
setCheckedDatasetTag,
onCreateCollectionTag,
isCreateCollectionTagLoading
} = useContextSelector(DatasetPageContext, (v) => v);
const { filterTags, setFilterTags, getData } = useContextSelector(
CollectionPageContext,
(v) => v
);
const checkedTags = filterTags;
const {
isOpen: isTagManageModalOpen,
onOpen: onOpenTagManageModal,
onClose: onCloseTagManageModal
} = useDisclosure();
const checkTags = (tag: DatasetTagType) => {
let currentCheckedTags = [];
if (checkedTags.includes(tag._id)) {
currentCheckedTags = checkedTags.filter((t) => t !== tag._id);
setCheckedDatasetTag(checkedDatasetTag.filter((t) => t._id !== tag._id));
} else {
currentCheckedTags = [...checkedTags, tag._id];
setCheckedDatasetTag([...checkedDatasetTag, tag]);
}
if (isEqual(currentCheckedTags, filterTags)) return;
setFilterTags(currentCheckedTags);
};
return (
<>
<MyPopover
placement="bottom"
hasArrow={false}
offset={[2, 2]}
w={'180px'}
closeOnBlur={true}
trigger={'click'}
Trigger={
<Flex
alignItems={'center'}
px={3}
py={2}
w={['140px', '180px']}
borderRadius={'md'}
border={'1px solid'}
borderColor={'myGray.250'}
cursor={'pointer'}
overflow={'hidden'}
h={['28px', '36px']}
fontSize={'sm'}
_hover={{
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
borderColor: 'primary.300'
}}
>
<Flex flex={'1 0 0'}>
{t('dataset:tag.tags')}
<Box as={'span'}>
{checkedTags.length > 0 && (
<Box ml={1} fontSize={'xs'} color={'myGray.600'}>
{`(${checkedTags.length})`}
</Box>
)}
</Box>
</Flex>
<MyIcon name={'core/chat/chevronDown'} w={'14px'} />
</Flex>
}
>
{({ onClose }) => (
<MyBox isLoading={isCreateCollectionTagLoading} onClick={(e) => e.stopPropagation()}>
<Box px={1.5} pt={1.5}>
<Input
pl={2}
h={8}
borderRadius={'xs'}
value={searchTagKey}
placeholder={t('dataset:tag.searchOrAddTag')}
onChange={(e) => setSearchTagKey(e.target.value)}
/>
</Box>
<Box my={1} px={1.5} maxH={'240px'} overflow={'auto'}>
{searchTagKey &&
!searchDatasetTagsResult.map((item) => item.tag).includes(searchTagKey) && (
<Flex
alignItems={'center'}
fontSize={'sm'}
px={1}
cursor={'pointer'}
_hover={{ bg: '#1118240D', color: 'primary.700' }}
borderRadius={'xs'}
onClick={() => onCreateCollectionTag(searchTagKey)}
>
<MyIcon name={'common/addLight'} w={'16px'} />
<Box ml={2} py={2}>
{t('dataset:tag.add') + ` "${searchTagKey}"`}
</Box>
</Flex>
)}
{[
...new Map(
[...checkedDatasetTag, ...searchDatasetTagsResult].map((item) => [item._id, item])
).values()
].map((item) => {
const checked = checkedTags.includes(item._id);
return (
<Flex
alignItems={'center'}
fontSize={'sm'}
px={1}
py={1}
my={1}
cursor={'pointer'}
color={checked ? 'primary.700' : 'myGray.600'}
_hover={{
bg: '#1118240D',
color: 'primary.700',
...(checked ? {} : { svg: { color: '#F3F3F4' } })
}}
borderRadius={'xs'}
key={item._id}
onClick={(e) => {
e.preventDefault();
checkTags(item);
}}
>
<Checkbox
isChecked={checkedTags.includes(item._id)}
onChange={(e) => {
checkTags(item);
}}
size={'md'}
icon={<MyIcon name={'common/check'} w={'12px'} />}
/>
<Box ml={2}>{item.tag}</Box>
</Flex>
);
})}
</Box>
<Flex borderTop={'1px solid #E8EBF0'} color={'myGray.600'}>
<Button
w={'full'}
fontSize={'sm'}
_hover={{ bg: '#1118240D', color: 'primary.700' }}
borderRadius={'none'}
borderBottomLeftRadius={'md'}
variant={'unstyled'}
onClick={() => {
setSearchTagKey('');
setFilterTags([]);
onClose();
}}
>
{t('dataset:tag.cancel')}
</Button>
<Box w={'1px'} bg={'myGray.200'}></Box>
<Button
w={'full'}
fontSize={'sm'}
_hover={{ bg: '#1118240D', color: 'primary.700' }}
borderRadius={'none'}
borderBottomRightRadius={'md'}
variant={'unstyled'}
onClick={() => {
onOpenTagManageModal();
}}
>
{t('dataset:tag.manage')}
</Button>
</Flex>
</MyBox>
)}
</MyPopover>
{isTagManageModalOpen && (
<TagManageModal
onClose={() => {
onCloseTagManageModal();
getData(1);
}}
/>
)}
</>
);
};
export default HeaderTagPopOver;

546
projects/app/src/pageComponents/dataset/detail/CollectionCard/TagManageModal.tsx Normal file
View File

@@ -0,0 +1,546 @@
import React, { useEffect, useMemo, useRef, useState } from 'react';
import { Input, Button, Flex, Box, Checkbox } from '@chakra-ui/react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { CollectionPageContext } from './Context';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import {
delDatasetCollectionTag,
getDatasetCollectionTags,
getScrollCollectionList,
getTagUsage,
postAddTagsToCollections,
updateDatasetCollectionTag
} from '@/web/core/dataset/api';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import MyInput from '@/components/MyInput';
import { DatasetTagType } from '@fastgpt/global/core/dataset/type';
import { ScrollListType, useVirtualScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm';
import { DatasetCollectionsListItemType } from '@/global/core/dataset/type';
const TagManageModal = ({ onClose }: { onClose: () => void }) => {
const { t } = useTranslation();
const {
datasetDetail,
onCreateCollectionTag,
isCreateCollectionTagLoading,
loadAllDatasetTags,
setSearchTagKey
} = useContextSelector(DatasetPageContext, (v) => v);
const { getData, pageNum, collections } = useContextSelector(CollectionPageContext, (v) => v);
const tagInputRef = useRef<HTMLInputElement>(null);
const editInputRef = useRef<HTMLInputElement>(null);
const [currentAddTag, setCurrentAddTag] = useState<
(DatasetTagType & { collections: string[] }) | undefined
>(undefined);
const [newTag, setNewTag] = useState<string | undefined>(undefined);
const [searchText, setSearchText] = useState('');
const [currentEditTagContent, setCurrentEditTagContent] = useState<string | undefined>(undefined);
const [currentEditTag, setCurrentEditTag] = useState<DatasetTagType | undefined>(undefined);
useEffect(() => {
if (newTag !== undefined && tagInputRef.current) {
tagInputRef.current?.focus();
}
}, [newTag]);
useEffect(() => {
if (currentEditTag !== undefined && editInputRef.current) {
editInputRef.current?.focus();
}
}, [currentEditTag]);
const { runAsync: onDeleteCollectionTag, loading: isDeleteCollectionTagLoading } = useRequest2(
(tag: string) =>
delDatasetCollectionTag({
datasetId: datasetDetail._id,
id: tag
}),
{
onSuccess() {
fetchData(1);
setSearchTagKey('');
loadAllDatasetTags();
},
successToast: t('common:common.Delete Success'),
errorToast: t('common:common.Delete Failed')
}
);
const { runAsync: onUpdateCollectionTag, loading: isUpdateCollectionTagLoading } = useRequest2(
async (tag: DatasetTagType) => {
return updateDatasetCollectionTag({
datasetId: datasetDetail._id,
tagId: tag._id,
tag: tag.tag
});
},
{
onSuccess() {
fetchData(1);
setSearchTagKey('');
loadAllDatasetTags();
}
}
);
const { runAsync: onSaveCollectionTag, loading: isSaveCollectionTagLoading } = useRequest2(
async ({
tag,
originCollectionIds,
collectionIds
}: {
tag: string;
originCollectionIds: string[];
collectionIds: string[];
}) => {
return postAddTagsToCollections({
tag,
originCollectionIds,
collectionIds,
datasetId: datasetDetail._id
});
},
{
onFinally() {
getData(pageNum);
},
successToast: t('common:common.Save Success'),
errorToast: t('common:common.Save Failed')
}
);
// Tags list
const {
scrollDataList: renderTags,
totalData: collectionTags,
ScrollList,
isLoading: isRequesting,
fetchData,
total: tagsTotal
} = useVirtualScrollPagination(getDatasetCollectionTags, {
refreshDeps: [''],
// debounceWait: 300,
itemHeight: 56,
overscan: 10,
pageSize: 10,
defaultParams: {
datasetId: datasetDetail._id,
searchText: ''
}
});
// Collections list
const {
scrollDataList: collectionsList,
ScrollList: ScrollListCollections,
isLoading: collectionsListLoading
} = useVirtualScrollPagination(getScrollCollectionList, {
refreshDeps: [searchText],
// debounceWait: 300,
itemHeight: 37,
overscan: 10,
pageSize: 30,
defaultParams: {
datasetId: datasetDetail._id,
searchText
}
});
const { data: tagUsages } = useRequest2(() => getTagUsage(datasetDetail._id), {
manual: false,
refreshDeps: [collections]
});
const isLoading =
isRequesting ||
isCreateCollectionTagLoading ||
isDeleteCollectionTagLoading ||
isUpdateCollectionTagLoading ||
isSaveCollectionTagLoading ||
collectionsListLoading;
return (
<MyModal
isOpen
onClose={onClose}
iconSrc="core/dataset/tag"
iconColor={'primary.600'}
title={t('dataset:tag.manage')}
w={'580px'}
h={'600px'}
closeOnOverlayClick={false}
isLoading={isLoading}
>
{currentAddTag === undefined ? (
<>
<Flex
alignItems={'center'}
color={'myGray.900'}
pb={2}
borderBottom={'1px solid #E8EBF0'}
mx={8}
pt={6}
>
<MyIcon name="menu" w={5} />
<Box ml={2} fontWeight={'semibold'} flex={'1 0 0'}>
{t('dataset:tag.total_tags', {
total: tagsTotal
})}
</Box>
<Button
size={'sm'}
leftIcon={<MyIcon name="common/addLight" w={4} />}
variant={'outline'}
fontSize={'xs'}
onClick={() => {
setNewTag('');
}}
>
{t('dataset:tag.Add New')}
</Button>
</Flex>
<Flex px={8} w={'full'}>
{newTag !== undefined && (
<Flex py={3} px={2} w={'full'} borderBottom={'1px solid #E8EBF0'}>
<Input
placeholder={t('dataset:tag.Add_new_tag')}
value={newTag}
onChange={(e) => setNewTag(e.target.value)}
ref={tagInputRef}
w={'200px'}
onBlur={async () => {
if (newTag && !collectionTags.map((item) => item.tag).includes(newTag)) {
await onCreateCollectionTag(newTag);
fetchData(1);
}
setNewTag(undefined);
}}
/>
</Flex>
)}
</Flex>
<ScrollList
px={8}
flex={'1 0 0'}
fontSize={'sm'}
EmptyChildren={<EmptyTip text={t('dataset:dataset.no_tags')} />}
>
{renderTags.map((listItem) => {
const item = listItem.data;
const tagUsage = tagUsages?.find((tagUsage) => tagUsage.tagId === item._id);
const collections = tagUsage?.collections || [];
const usage = collections.length;
return (
<Flex
py={2}
borderBottom={'1px solid #E8EBF0'}
sx={{
'&:hover .icon-box': {
display: 'flex'
}
}}
key={item._id}
>
<Flex
px={2}
py={1}
flex={'1'}
_hover={{ bg: 'myGray.100' }}
alignItems={'center'}
borderRadius={'xs'}
>
<Flex
flex={'1 0 0'}
alignItems={'center'}
onClick={() => {
setCurrentAddTag({ ...item, collections });
}}
cursor={'pointer'}
>
{currentEditTag?._id !== item._id ? (
<Box
px={3}
py={1.5}
bg={'#DBF3FF'}
color={'#0884DD'}
fontSize={'xs'}
borderRadius={'sm'}
>
{item.tag}
</Box>
) : (
<Input
placeholder={t('dataset:tag.Edit_tag')}
value={
currentEditTagContent !== undefined ? currentEditTagContent : item.tag
}
onChange={(e) => setCurrentEditTagContent(e.target.value)}
ref={editInputRef}
w={'200px'}
onBlur={() => {
if (
currentEditTagContent &&
!collectionTags
.map((item) => item.tag)
.includes(currentEditTagContent)
) {
onUpdateCollectionTag({
tag: currentEditTagContent,
_id: item._id
});
}
setCurrentEditTag(undefined);
setCurrentEditTagContent(undefined);
}}
/>
)}
<Box as={'span'} color={'myGray.500'} ml={2}>{`(${usage})`}</Box>
</Flex>
<Box
className="icon-box"
display="none"
_hover={{ bg: '#1118240D' }}
mr={2}
p={1}
borderRadius={'sm'}
onClick={() => {
setCurrentAddTag({ ...item, collections });
}}
cursor={'pointer'}
>
<MyIcon name="common/add2" w={4} />
</Box>
<Box
className="icon-box"
display="none"
_hover={{ bg: '#1118240D' }}
mr={2}
p={1}
borderRadius={'sm'}
cursor={'pointer'}
onClick={(e) => {
setCurrentEditTag(item);
editInputRef.current?.focus();
}}
>
<MyIcon name="edit" w={4} />
</Box>
<PopoverConfirm
showCancel
content={t('dataset:tag.delete_tag_confirm')}
type="delete"
Trigger={
<Box
className="icon-box"
display="none"
_hover={{ bg: '#1118240D' }}
p={1}
borderRadius={'sm'}
cursor={'pointer'}
>
<MyIcon name="delete" w={4} />
</Box>
}
onConfirm={() => onDeleteCollectionTag(item._id)}
/>
</Flex>
</Flex>
);
})}
</ScrollList>
</>
) : (
<AddTagToCollections
currentAddTag={currentAddTag}
setCurrentAddTag={setCurrentAddTag}
onSaveCollectionTag={onSaveCollectionTag}
setSearchText={setSearchText}
collectionsList={collectionsList}
ScrollListCollections={ScrollListCollections}
/>
)}
</MyModal>
);
};
export default TagManageModal;
const AddTagToCollections = ({
currentAddTag,
setCurrentAddTag,
onSaveCollectionTag,
setSearchText,
collectionsList,
ScrollListCollections
}: {
currentAddTag: DatasetTagType & { collections: string[] };
setCurrentAddTag: (tag: (DatasetTagType & { collections: string[] }) | undefined) => void;
onSaveCollectionTag: ({
tag,
originCollectionIds,
collectionIds
}: {
tag: string;
originCollectionIds: string[];
collectionIds: string[];
}) => void;
setSearchText: (text: string) => void;
collectionsList: {
index: number;
data: DatasetCollectionsListItemType;
}[];
ScrollListCollections: ScrollListType;
}) => {
const { t } = useTranslation();
const [selectedCollections, setSelectedCollections] = useState<string[]>(
currentAddTag.collections
);
const [originCollections, setOriginCollections] = useState<string[]>(currentAddTag.collections);
const formatCollections = useMemo(
() =>
collectionsList.map((item) => {
const collection = item.data;
const icon = getCollectionIcon(collection.type, collection.name);
return {
id: collection._id,
tags: collection.tags,
name: collection.name,
icon
};
}),
[collectionsList]
);
return (
<>
<Flex alignItems={'center'} pb={2} mx={8} pt={6} borderBottom={'1px solid #E8EBF0'}>
<MyIcon
name="common/backFill"
w={4}
cursor={'pointer'}
onClick={() => {
setCurrentAddTag(undefined);
setSearchText('');
}}
/>
{
<Flex alignItems={'center'}>
<Box
ml={2}
px={3}
py={1.5}
bg={'#DBF3FF'}
color={'#0884DD'}
fontSize={'sm'}
borderRadius={'sm'}
>
{currentAddTag.tag}
</Box>
<Box
as={'span'}
fontSize={'sm'}
color={'myGray.500'}
ml={2}
>{`(${selectedCollections.length})`}</Box>
</Flex>
}
<Box flex={'1 0 0'}></Box>
<MyInput
placeholder={t('common:common.Search')}
w={'200px'}
mr={2}
onChange={(e) => {
setSearchText(e.target.value);
}}
/>
<Button
leftIcon={<MyIcon name="save" w={4} />}
onClick={() => {
onSaveCollectionTag({
tag: currentAddTag._id,
originCollectionIds: originCollections,
collectionIds: selectedCollections
});
setOriginCollections(selectedCollections);
}}
>
{t('common:common.Save')}
</Button>
</Flex>
<ScrollListCollections
px={8}
mt={2}
flex={'1 0 0'}
fontSize={'sm'}
EmptyChildren={<EmptyTip text={t('dataset:dataset.no_collections')} />}
>
{formatCollections.map((collection) => {
return (
<Flex
px={2}
py={1}
mb={2}
flex={'1'}
_hover={{
bg: 'myGray.100',
...(!selectedCollections.includes(collection.id)
? { svg: { color: 'myGray.100' } }
: {})
}}
alignItems={'center'}
borderRadius={'xs'}
key={collection.id}
cursor={'pointer'}
onClick={() => {
setSelectedCollections((prev) => {
if (prev.includes(collection.id)) {
return prev.filter((id) => id !== collection.id);
} else {
return [...prev, collection.id];
}
});
}}
>
<Checkbox
size={'md'}
mr={2}
icon={<MyIcon name="common/check" w={'12px'} />}
onChange={() => {
setSelectedCollections((prev) => {
if (prev.includes(collection.id)) {
return prev.filter((id) => id !== collection.id);
} else {
return [...prev, collection.id];
}
});
}}
isChecked={selectedCollections.includes(collection.id)}
/>
<MyIcon name={collection.icon as any} w={'20px'} mr={2} />
<Box fontSize={'sm'} borderRadius={'sm'} color={'myGray.900'}>
{collection.name}
</Box>
</Flex>
);
})}
</ScrollListCollections>
</>
);
};

262
projects/app/src/pageComponents/dataset/detail/CollectionCard/TagsPopOver.tsx Normal file
View File

@@ -0,0 +1,262 @@
import { Box, Checkbox, Flex, Input } from '@chakra-ui/react';
import MyPopover from '@fastgpt/web/components/common/MyPopover';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { putDatasetCollectionById } from '@/web/core/dataset/api';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useTranslation } from 'next-i18next';
import { useMemo, useRef, useState } from 'react';
import { useDeepCompareEffect } from 'ahooks';
import { DatasetCollectionItemType, DatasetTagType } from '@fastgpt/global/core/dataset/type';
import { isEqual } from 'lodash';
import { DatasetCollectionsListItemType } from '@/global/core/dataset/type';
const TagsPopOver = ({
currentCollection
}: {
currentCollection: DatasetCollectionItemType | DatasetCollectionsListItemType;
}) => {
const { t } = useTranslation();
const {
searchTagKey,
setSearchTagKey,
searchDatasetTagsResult,
allDatasetTags,
onCreateCollectionTag,
isCreateCollectionTagLoading
} = useContextSelector(DatasetPageContext, (v) => v);
const [collectionTags, setCollectionTags] = useState<string[]>(currentCollection.tags ?? []);
const [checkedTags, setCheckedTags] = useState<DatasetTagType[]>([]);
const [showTagManage, setShowTagManage] = useState(false);
const [isUpdateLoading, setIsUpdateLoading] = useState(false);
const tagList = useMemo(
() =>
(collectionTags
?.map((item) => {
const tagObject = allDatasetTags.find((tag) => tag.tag === item);
return tagObject ? { _id: tagObject._id, tag: tagObject.tag } : null;
})
.filter((tag) => tag !== null) as {
_id: string;
tag: string;
}[]) || [],
[collectionTags, allDatasetTags]
);
const [visibleTags, setVisibleTags] = useState<DatasetTagType[]>(tagList);
const [overflowTags, setOverflowTags] = useState<DatasetTagType[]>([]);
const containerRef = useRef<HTMLDivElement>(null);
useDeepCompareEffect(() => {
const calculateTags = () => {
if (!containerRef.current || !tagList) return;
const containerWidth = containerRef.current.offsetWidth;
const tagWidth = 11;
let totalWidth = 30;
let visibleCount = 0;
for (let i = 0; i < tagList.length; i++) {
const tag = tagList[i];
const estimatedWidth = tag.tag.length * tagWidth + 16; // 加上左右 padding 的宽度
if (totalWidth + estimatedWidth <= containerWidth) {
totalWidth += estimatedWidth;
visibleCount++;
} else {
break;
}
}
setVisibleTags(tagList.slice(0, visibleCount));
setOverflowTags(tagList.slice(visibleCount));
};
setTimeout(calculateTags, 100);
setCheckedTags(tagList);
window.addEventListener('resize', calculateTags);
return () => {
window.removeEventListener('resize', calculateTags);
};
}, [tagList]);
return (
<MyPopover
placement={showTagManage ? 'bottom' : 'bottom-end'}
hasArrow={false}
offset={[2, 2]}
w={'180px'}
trigger={'hover'}
Trigger={
<MyBox
ref={containerRef}
display={'flex'}
isLoading={isUpdateLoading}
size={'xs'}
mt={1}
py={0.5}
px={0.25}
_hover={{
bg: 'myGray.50',
borderRadius: '3px'
}}
onMouseEnter={(e) => {
e.stopPropagation();
if (!e.currentTarget.parentElement || !e.currentTarget.parentElement.parentElement)
return;
e.currentTarget.parentElement.parentElement.style.backgroundColor = 'white';
}}
onMouseLeave={(e) => {
if (!e.currentTarget.parentElement || !e.currentTarget.parentElement.parentElement)
return;
e.currentTarget.parentElement.parentElement.style.backgroundColor = '';
}}
onClick={(e) => {
e.stopPropagation();
setShowTagManage(true);
}}
cursor={'pointer'}
>
<Flex>
{visibleTags.map((item, index) => (
<Box
key={index}
h={5}
mr={2}
px={2}
fontSize={'11px'}
fontWeight={'500'}
bg={'#F0FBFF'}
color={'#0884DD'}
borderRadius={'xs'}
>
{item.tag}
</Box>
))}
</Flex>
{overflowTags.length > 0 && (
<Box h={5} px={2} bg={'#1118240D'} borderRadius={'33px'} fontSize={'11px'}>
{`+${overflowTags.length}`}
</Box>
)}
</MyBox>
}
onCloseFunc={async () => {
setSearchTagKey('');
setShowTagManage(false);
if (isEqual(checkedTags, tagList) || !showTagManage) return;
setIsUpdateLoading(true);
await putDatasetCollectionById({
id: currentCollection._id,
tags: checkedTags.map((tag) => tag.tag)
});
setCollectionTags(checkedTags.map((tag) => tag.tag));
setIsUpdateLoading(false);
}}
display={showTagManage || overflowTags.length > 0 ? 'block' : 'none'}
>
{({}) => (
<>
{showTagManage ? (
<MyBox isLoading={isCreateCollectionTagLoading} onClick={(e) => e.stopPropagation()}>
<Box px={1.5} pt={1.5}>
<Input
pl={2}
h={7}
borderRadius={'xs'}
value={searchTagKey}
placeholder={t('dataset:tag.searchOrAddTag')}
onChange={(e) => setSearchTagKey(e.target.value)}
/>
</Box>
<Box my={1} px={1.5} maxH={'200px'} overflow={'auto'}>
{searchTagKey &&
!searchDatasetTagsResult.map((item) => item.tag).includes(searchTagKey) && (
<Flex
alignItems={'center'}
fontSize={'xs'}
px={1}
cursor={'pointer'}
_hover={{ bg: '#1118240D', color: '#2B5FD9' }}
borderRadius={'xs'}
onClick={() => onCreateCollectionTag(searchTagKey)}
>
<MyIcon name={'common/addLight'} w={'1rem'} />
<Box ml={1} py={1}>
{t('dataset:tag.add') + ` "${searchTagKey}"`}
</Box>
</Flex>
)}
{searchDatasetTagsResult?.map((item) => {
const tagsList = checkedTags.map((tag) => tag.tag);
return (
<Flex
alignItems={'center'}
fontSize={'xs'}
px={1}
py={1}
my={1}
key={item._id}
cursor={'pointer'}
color={tagsList.includes(item.tag) ? '#2B5FD9' : 'myGray.600'}
_hover={{
bg: '#1118240D',
color: '#2B5FD9',
...(tagsList.includes(item.tag) ? {} : { svg: { color: '#F3F3F4' } })
}}
borderRadius={'xs'}
onClick={(e) => {
e.preventDefault();
if (tagsList.includes(item.tag)) {
setCheckedTags(checkedTags.filter((t) => t.tag !== item.tag));
} else {
setCheckedTags([...checkedTags, item]);
}
}}
>
<Checkbox
isChecked={tagsList.includes(item.tag)}
onChange={(e) => {
if (e.target.checked) {
setCheckedTags([...checkedTags, item]);
} else {
setCheckedTags(checkedTags.filter((t) => t._id !== item._id));
}
}}
icon={<MyIcon name={'common/check'} w={'12px'} />}
/>
<Box ml={2}>{item.tag}</Box>
</Flex>
);
})}
</Box>
</MyBox>
) : (
<Flex gap={1} p={3} flexWrap={'wrap'}>
{overflowTags.map((tag, index) => (
<Box
key={index}
h={5}
px={2}
fontSize={'11px'}
bg={'#F0FBFF'}
color={'#0884DD'}
borderRadius={'xs'}
>
{tag.tag}
</Box>
))}
</Flex>
)}
</>
)}
</MyPopover>
);
};
export default TagsPopOver;

114
projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx Normal file
View File

@@ -0,0 +1,114 @@
import React from 'react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useTranslation } from 'next-i18next';
import { Box, Button, Input, Link, ModalBody, ModalFooter } from '@chakra-ui/react';
import { strIsLink } from '@fastgpt/global/common/string/tools';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { getDocPath } from '@/web/common/system/doc';
import { useSystemStore } from '@/web/common/system/useSystemStore';
type FormType = {
url?: string | undefined;
selector?: string | undefined;
};
const WebsiteConfigModal = ({
onClose,
onSuccess,
defaultValue = {
url: '',
selector: ''
}
}: {
onClose: () => void;
onSuccess: (data: FormType) => void;
defaultValue?: FormType;
}) => {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const { toast } = useToast();
const { register, handleSubmit } = useForm({
defaultValues: defaultValue
});
const isEdit = !!defaultValue.url;
const confirmTip = isEdit
? t('common:core.dataset.website.Confirm Update Tips')
: t('common:core.dataset.website.Confirm Create Tips');
const { ConfirmModal, openConfirm } = useConfirm({
type: 'common'
});
return (
<MyModal
isOpen
iconSrc="core/dataset/websiteDataset"
title={t('common:core.dataset.website.Config')}
onClose={onClose}
maxW={'500px'}
>
<ModalBody>
<Box fontSize={'sm'} color={'myGray.600'}>
{t('common:core.dataset.website.Config Description')}
{feConfigs?.docUrl && (
<Link
href={getDocPath('/docs/guide/knowledge_base/websync/')}
target="_blank"
textDecoration={'underline'}
fontWeight={'bold'}
>
{t('common:common.course.Read Course')}
</Link>
)}
</Box>
<Box mt={2}>
<Box>{t('common:core.dataset.website.Base Url')}</Box>
<Input
placeholder={t('common:core.dataset.collection.Website Link')}
{...register('url', {
required: true
})}
/>
</Box>
<Box mt={3}>
<Box>
{t('common:core.dataset.website.Selector')}({t('common:common.choosable')})
</Box>
<Input {...register('selector')} placeholder="body .content #document" />
</Box>
</ModalBody>
<ModalFooter>
<Button variant={'whiteBase'} onClick={onClose}>
{t('common:common.Close')}
</Button>
<Button
ml={2}
onClick={handleSubmit((data) => {
if (!data.url) return;
// check is link
if (!strIsLink(data.url)) {
return toast({
status: 'warning',
title: t('common:common.link.UnValid')
});
}
openConfirm(
() => {
onSuccess(data);
},
undefined,
confirmTip
)();
})}
>
{t('common:core.dataset.website.Start Sync')}
</Button>
</ModalFooter>
<ConfirmModal />
</MyModal>
);
};
export default WebsiteConfigModal;

436
projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx Normal file
View File

@@ -0,0 +1,436 @@
import React, { useState, useRef, useMemo } from 'react';
import {
Box,
Flex,
TableContainer,
Table,
Thead,
Tr,
Th,
Td,
Tbody,
MenuButton,
Switch
} from '@chakra-ui/react';
import {
delDatasetCollectionById,
putDatasetCollectionById,
postLinkCollectionSync
} from '@/web/core/dataset/api';
import { useQuery } from '@tanstack/react-query';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useRouter } from 'next/router';
import MyMenu from '@fastgpt/web/components/common/MyMenu';
import { useEditTitle } from '@/web/common/hooks/useEditTitle';
import {
DatasetCollectionTypeEnum,
DatasetStatusEnum,
DatasetCollectionSyncResultMap,
DatasetTypeEnum
} from '@fastgpt/global/core/dataset/constants';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
import { TabEnum } from '../../../../pages/dataset/detail/index';
import dynamic from 'next/dynamic';
import SelectCollections from '@/web/core/dataset/components/SelectCollections';
import { useToast } from '@fastgpt/web/hooks/useToast';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { DatasetCollectionSyncResultEnum } from '@fastgpt/global/core/dataset/constants';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useContextSelector } from 'use-context-selector';
import { CollectionPageContext } from './Context';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import {
checkCollectionIsFolder,
getTrainingTypeLabel
} from '@fastgpt/global/core/dataset/collection/utils';
import { useFolderDrag } from '@/components/common/folder/useFolderDrag';
import TagsPopOver from './TagsPopOver';
import { useSystemStore } from '@/web/common/system/useSystemStore';
const Header = dynamic(() => import('./Header'));
const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip'));
const CollectionCard = () => {
const BoxRef = useRef<HTMLDivElement>(null);
const router = useRouter();
const { toast } = useToast();
const { t } = useTranslation();
const { datasetDetail, loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v);
const { feConfigs } = useSystemStore();
const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
content: t('common:dataset.Confirm to delete the file'),
type: 'delete'
});
const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('common:Rename')
});
const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>();
const { collections, Pagination, total, getData, isGetting, pageNum, pageSize } =
useContextSelector(CollectionPageContext, (v) => v);
// Ad file status icon
const formatCollections = useMemo(
() =>
collections.map((collection) => {
const icon = getCollectionIcon(collection.type, collection.name);
const status = (() => {
if (collection.trainingAmount > 0) {
return {
statusText: t('common:dataset.collections.Collection Embedding', {
total: collection.trainingAmount
}),
colorSchema: 'gray'
};
}
return {
statusText: t('common:core.dataset.collection.status.active'),
colorSchema: 'green'
};
})();
return {
...collection,
icon,
...status
};
}),
[collections, t]
);
const { runAsync: onUpdateCollection, loading: isUpdating } = useRequest2(
putDatasetCollectionById,
{
onSuccess() {
getData(pageNum);
},
successToast: t('common:common.Update Success')
}
);
const { runAsync: onDelCollection, loading: isDeleting } = useRequest2(
(collectionId: string) => {
return delDatasetCollectionById({
id: collectionId
});
},
{
onSuccess() {
getData(pageNum);
},
successToast: t('common:common.Delete Success'),
errorToast: t('common:common.Delete Failed')
}
);
const { openConfirm: openSyncConfirm, ConfirmModal: ConfirmSyncModal } = useConfirm({
content: t('dataset:collection_sync_confirm_tip')
});
const { runAsync: onclickStartSync, loading: isSyncing } = useRequest2(postLinkCollectionSync, {
onSuccess(res: DatasetCollectionSyncResultEnum) {
getData(pageNum);
toast({
status: 'success',
title: t(DatasetCollectionSyncResultMap[res]?.label as any)
});
},
errorToast: t('common:core.dataset.error.Start Sync Failed')
});
const hasTrainingData = useMemo(
() => !!formatCollections.find((item) => item.trainingAmount > 0),
[formatCollections]
);
useQuery(
['refreshCollection'],
() => {
getData(pageNum);
if (datasetDetail.status === DatasetStatusEnum.syncing) {
loadDatasetDetail(datasetDetail._id);
}
return null;
},
{
refetchInterval: 6000,
enabled: hasTrainingData || datasetDetail.status === DatasetStatusEnum.syncing
}
);
const { getBoxProps, isDropping } = useFolderDrag({
activeStyles: {
bg: 'primary.100'
},
onDrop: async (dragId: string, targetId: string) => {
try {
await putDatasetCollectionById({
id: dragId,
parentId: targetId
});
getData(pageNum);
} catch (error) {}
}
});
const isLoading =
isUpdating || isDeleting || isSyncing || (isGetting && collections.length === 0) || isDropping;
return (
<MyBox isLoading={isLoading} h={'100%'} py={[2, 4]}>
<Flex ref={BoxRef} flexDirection={'column'} py={[1, 0]} h={'100%'} px={[2, 6]}>
{/* header */}
<Header />
{/* collection table */}
<TableContainer mt={3} overflowY={'auto'} fontSize={'sm'}>
<Table variant={'simple'} draggable={false}>
<Thead draggable={false}>
<Tr>
<Th py={4}>{t('common:common.Name')}</Th>
<Th py={4}>{t('dataset:collection.Training type')}</Th>
<Th py={4}>{t('common:dataset.collections.Data Amount')}</Th>
<Th py={4}>{t('dataset:collection.Create update time')}</Th>
<Th py={4}>{t('common:common.Status')}</Th>
<Th py={4}>{t('dataset:Enable')}</Th>
<Th py={4} />
</Tr>
</Thead>
<Tbody>
<Tr h={'5px'} />
{formatCollections.map((collection) => (
<Tr
key={collection._id}
_hover={{ bg: 'myGray.50' }}
cursor={'pointer'}
{...getBoxProps({
dataId: collection._id,
isFolder: collection.type === DatasetCollectionTypeEnum.folder
})}
draggable={false}
onClick={() => {
if (collection.type === DatasetCollectionTypeEnum.folder) {
router.push({
query: {
datasetId: datasetDetail._id,
parentId: collection._id
}
});
} else {
router.push({
query: {
datasetId: datasetDetail._id,
collectionId: collection._id,
currentTab: TabEnum.dataCard
}
});
}
}}
>
<Td minW={'150px'} maxW={['200px', '300px']} draggable py={2}>
<Flex alignItems={'center'}>
<MyIcon name={collection.icon as any} w={'1.25rem'} mr={2} />
<MyTooltip
label={t('common:common.folder.Drag Tip')}
shouldWrapChildren={false}
>
<Box color={'myGray.900'} fontWeight={'500'} className="textEllipsis">
{collection.name}
</Box>
</MyTooltip>
</Flex>
{feConfigs?.isPlus && !!collection.tags?.length && (
<TagsPopOver currentCollection={collection} />
)}
</Td>
<Td py={2}>
{!checkCollectionIsFolder(collection.type) ? (
<>{t((getTrainingTypeLabel(collection.trainingType) || '-') as any)}</>
) : (
'-'
)}
</Td>
<Td py={2}>{collection.dataAmount || '-'}</Td>
<Td fontSize={'xs'} py={2} color={'myGray.500'}>
<Box>{formatTime2YMDHM(collection.createTime)}</Box>
<Box>{formatTime2YMDHM(collection.updateTime)}</Box>
</Td>
<Td py={2}>
<MyTag showDot colorSchema={collection.colorSchema as any} type={'borderFill'}>
{t(collection.statusText as any)}
</MyTag>
</Td>
<Td py={2} onClick={(e) => e.stopPropagation()}>
<Switch
isChecked={!collection.forbid}
size={'sm'}
onChange={(e) =>
onUpdateCollection({
id: collection._id,
forbid: !e.target.checked
})
}
/>
</Td>
<Td py={2} onClick={(e) => e.stopPropagation()}>
{collection.permission.hasWritePer && (
<MyMenu
width={100}
offset={[-70, 5]}
Button={
<MenuButton
w={'1.5rem'}
h={'1.5rem'}
borderRadius={'md'}
_hover={{
color: 'primary.500',
'& .icon': {
bg: 'myGray.200'
}
}}
>
<MyIcon
className="icon"
name={'more'}
h={'1rem'}
w={'1rem'}
px={1}
py={1}
borderRadius={'md'}
cursor={'pointer'}
/>
</MenuButton>
}
menuList={[
{
children: [
...(collection.type === DatasetCollectionTypeEnum.link ||
datasetDetail.type === DatasetTypeEnum.apiDataset
? [
{
label: (
<Flex alignItems={'center'}>
<MyIcon
name={'common/refreshLight'}
w={'0.9rem'}
mr={2}
/>
{t('dataset:collection_sync')}
</Flex>
),
onClick: () =>
openSyncConfirm(() => {
onclickStartSync(collection._id);
})()
}
]
: []),
{
label: (
<Flex alignItems={'center'}>
<MyIcon name={'common/file/move'} w={'0.9rem'} mr={2} />
{t('common:Move')}
</Flex>
),
onClick: () =>
setMoveCollectionData({ collectionId: collection._id })
},
{
label: (
<Flex alignItems={'center'}>
<MyIcon name={'edit'} w={'0.9rem'} mr={2} />
{t('common:Rename')}
</Flex>
),
onClick: () =>
onOpenEditTitleModal({
defaultVal: collection.name,
onSuccess: (newName) =>
onUpdateCollection({
id: collection._id,
name: newName
})
})
}
]
},
{
children: [
{
label: (
<Flex alignItems={'center'}>
<MyIcon
mr={1}
name={'delete'}
w={'0.9rem'}
_hover={{ color: 'red.600' }}
/>
<Box>{t('common:common.Delete')}</Box>
</Flex>
),
type: 'danger',
onClick: () =>
openDeleteConfirm(
() => {
onDelCollection(collection._id);
},
undefined,
collection.type === DatasetCollectionTypeEnum.folder
? t('common:dataset.collections.Confirm to delete the folder')
: t('common:dataset.Confirm to delete the file')
)()
}
]
}
]}
/>
)}
</Td>
</Tr>
))}
</Tbody>
</Table>
{total > pageSize && (
<Flex mt={2} justifyContent={'center'}>
<Pagination />
</Flex>
)}
{total === 0 && <EmptyCollectionTip />}
</TableContainer>
<ConfirmDeleteModal />
<ConfirmSyncModal />
<EditTitleModal />
{!!moveCollectionData && (
<SelectCollections
datasetId={datasetDetail._id}
type="folder"
defaultSelectedId={[moveCollectionData.collectionId]}
onClose={() => setMoveCollectionData(undefined)}
onSuccess={async ({ parentId }) => {
await putDatasetCollectionById({
id: moveCollectionData.collectionId,
parentId
});
getData(pageNum);
setMoveCollectionData(undefined);
toast({
status: 'success',
title: t('common:common.folder.Move Success')
});
}}
/>
)}
</Flex>
</MyBox>
);
};
export default React.memo(CollectionCard);

364
projects/app/src/pageComponents/dataset/detail/DataCard.tsx Normal file
View File

@@ -0,0 +1,364 @@
import React, { useState, useMemo } from 'react';
import { Box, Card, IconButton, Flex, Button, useTheme } from '@chakra-ui/react';
import {
getDatasetDataList,
delOneDatasetDataById,
getDatasetCollectionById
} from '@/web/core/dataset/api';
import { useQuery } from '@tanstack/react-query';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { useTranslation } from 'next-i18next';
import { useRouter } from 'next/router';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyInput from '@/components/MyInput';
import InputDataModal from './InputDataModal';
import RawSourceBox from '@/components/core/dataset/RawSourceBox';
import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useContextSelector } from 'use-context-selector';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import TagsPopOver from './CollectionCard/TagsPopOver';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyDivider from '@fastgpt/web/components/common/MyDivider';
import Markdown from '@/components/Markdown';
import { useMemoizedFn } from 'ahooks';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import { TabEnum } from './NavBar';
import {
DatasetCollectionTypeEnum,
ImportDataSourceEnum
} from '@fastgpt/global/core/dataset/constants';
const DataCard = () => {
const theme = useTheme();
const router = useRouter();
const { isPc } = useSystem();
const { collectionId = '', datasetId } = router.query as {
collectionId: string;
datasetId: string;
};
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { feConfigs } = useSystemStore();
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
const { toast } = useToast();
const scrollParams = useMemo(
() => ({
collectionId,
searchText
}),
[collectionId, searchText]
);
const EmptyTipDom = useMemo(
() => <EmptyTip text={t('common:core.dataset.data.Empty Tip')} />,
[t]
);
const {
data: datasetDataList,
ScrollData,
total,
refreshList,
setData: setDatasetDataList
} = useScrollPagination(getDatasetDataList, {
pageSize: 15,
params: scrollParams,
refreshDeps: [searchText, collectionId],
EmptyTip: EmptyTipDom
});
const [editDataId, setEditDataId] = useState<string>();
// get file info
const { data: collection } = useQuery(
['getDatasetCollectionById', collectionId],
() => getDatasetCollectionById(collectionId),
{
onError: () => {
router.replace({
query: {
datasetId
}
});
}
}
);
const canWrite = useMemo(() => datasetDetail.permission.hasWritePer, [datasetDetail]);
const { openConfirm, ConfirmModal } = useConfirm({
content: t('common:dataset.Confirm to delete the data'),
type: 'delete'
});
const onDeleteOneData = useMemoizedFn((dataId: string) => {
openConfirm(async () => {
try {
await delOneDatasetDataById(dataId);
setDatasetDataList((prev) => {
return prev.filter((data) => data._id !== dataId);
});
toast({
title: t('common:common.Delete Success'),
status: 'success'
});
} catch (error) {
toast({
title: getErrText(error),
status: 'error'
});
}
})();
});
return (
<MyBox py={[1, 0]} h={'100%'}>
<Flex flexDirection={'column'} h={'100%'}>
{/* Header */}
<Flex alignItems={'center'} px={6}>
<Box flex={'1 0 0'} mr={[3, 5]} alignItems={'center'}>
<Box
className="textEllipsis"
alignItems={'center'}
gap={2}
display={isPc ? 'flex' : ''}
>
{collection?._id && (
<RawSourceBox
collectionId={collection._id}
{...getCollectionSourceData(collection)}
fontSize={['sm', 'md']}
color={'black'}
textDecoration={'none'}
/>
)}
</Box>
{feConfigs?.isPlus && !!collection?.tags?.length && (
<TagsPopOver currentCollection={collection} />
)}
</Box>
{datasetDetail.type !== 'websiteDataset' && !!collection?.chunkSize && (
<Button
ml={2}
variant={'whitePrimary'}
size={['sm', 'md']}
onClick={() => {
router.push({
query: {
datasetId,
currentTab: TabEnum.import,
source: ImportDataSourceEnum.reTraining,
collectionId
}
});
}}
>
{t('dataset:retain_collection')}
</Button>
)}
{canWrite && (
<Button
ml={2}
variant={'whitePrimary'}
size={['sm', 'md']}
isDisabled={!collection}
onClick={() => {
setEditDataId('');
}}
>
{t('common:dataset.Insert Data')}
</Button>
)}
</Flex>
<Box justifyContent={'center'} px={6} pos={'relative'} w={'100%'}>
<MyDivider my={'17px'} w={'100%'} />
</Box>
<Flex alignItems={'center'} px={6} pb={4}>
<Flex align={'center'} color={'myGray.500'}>
<MyIcon name="common/list" mr={2} w={'18px'} />
<Box as={'span'} fontSize={['sm', '14px']} fontWeight={'500'}>
{t('common:core.dataset.data.Total Amount', { total })}
</Box>
</Flex>
<Box flex={1} mr={1} />
<MyInput
leftIcon={
<MyIcon
name="common/searchLight"
position={'absolute'}
w={'14px'}
color={'myGray.600'}
/>
}
bg={'myGray.25'}
borderColor={'myGray.200'}
color={'myGray.500'}
w={['200px', '300px']}
placeholder={t('common:core.dataset.data.Search data placeholder')}
value={searchText}
onChange={(e) => {
setSearchText(e.target.value);
}}
/>
</Flex>
{/* data */}
<ScrollData px={5} pb={5}>
<Flex flexDir={'column'} gap={2}>
{datasetDataList.map((item, index) => (
<Card
key={item._id}
cursor={'pointer'}
p={3}
userSelect={'none'}
boxShadow={'none'}
bg={index % 2 === 1 ? 'myGray.50' : 'blue.50'}
border={theme.borders.sm}
position={'relative'}
overflow={'hidden'}
_hover={{
borderColor: 'blue.600',
boxShadow: 'lg',
'& .header': { visibility: 'visible' },
'& .footer': { visibility: 'visible' },
bg: index % 2 === 1 ? 'myGray.200' : 'blue.100'
}}
onClick={(e) => {
e.stopPropagation();
setEditDataId(item._id);
}}
>
{/* Data tag */}
<Flex
position={'absolute'}
zIndex={1}
alignItems={'center'}
visibility={'hidden'}
className="header"
>
<MyTag
px={2}
type="borderFill"
borderRadius={'sm'}
border={'1px'}
color={'myGray.200'}
bg={'white'}
fontWeight={'500'}
>
<Box color={'blue.600'}>#{item.chunkIndex ?? '-'} </Box>
<Box
ml={1.5}
className={'textEllipsis'}
fontSize={'mini'}
textAlign={'right'}
color={'myGray.500'}
>
ID:{item._id}
</Box>
</MyTag>
</Flex>
{/* Data content */}
<Box wordBreak={'break-all'} fontSize={'sm'}>
<Markdown source={item.q} isDisabled />
{!!item.a && (
<>
<MyDivider />
<Markdown source={item.a} isDisabled />
</>
)}
</Box>
{/* Mask */}
<Flex
className="footer"
position={'absolute'}
bottom={2}
right={2}
overflow={'hidden'}
alignItems={'flex-end'}
visibility={'hidden'}
fontSize={'mini'}
>
<Flex
alignItems={'center'}
bg={'white'}
color={'myGray.600'}
borderRadius={'sm'}
border={'1px'}
borderColor={'myGray.200'}
h={'24px'}
px={2}
fontSize={'mini'}
boxShadow={'1'}
py={1}
mr={2}
>
<MyIcon
bg={'white'}
color={'myGray.600'}
borderRadius={'sm'}
border={'1px'}
borderColor={'myGray.200'}
name="common/text/t"
w={'14px'}
mr={1}
/>
{item.q.length + (item.a?.length || 0)}
</Flex>
{canWrite && (
<IconButton
display={'flex'}
p={1}
boxShadow={'1'}
icon={<MyIcon name={'common/trash'} w={'14px'} color={'myGray.600'} />}
variant={'whiteDanger'}
size={'xsSquare'}
onClick={(e) => {
e.stopPropagation();
onDeleteOneData(item._id);
}}
aria-label={''}
/>
)}
</Flex>
</Card>
))}
</Flex>
</ScrollData>
</Flex>
{editDataId !== undefined && collection && (
<InputDataModal
collectionId={collection._id}
dataId={editDataId}
onClose={() => setEditDataId(undefined)}
onSuccess={(data) => {
if (editDataId === '') {
refreshList();
return;
}
setDatasetDataList((prev) => {
return prev.map((item) => {
if (item._id === editDataId) {
return {
...item,
...data
};
}
return item;
});
});
}}
/>
)}
<ConfirmModal />
</MyBox>
);
};
export default React.memo(DataCard);

317
projects/app/src/pageComponents/dataset/detail/Import/Context.tsx Normal file
View File

@@ -0,0 +1,317 @@
import { useRouter } from 'next/router';
import { SetStateAction, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { createContext, useContextSelector } from 'use-context-selector';
import { ImportDataSourceEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { useMyStep } from '@fastgpt/web/hooks/useStep';
import { Box, Button, Flex, IconButton } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { TabEnum } from '../NavBar';
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
import { UseFormReturn, useForm } from 'react-hook-form';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
type TrainingFiledType = {
chunkOverlapRatio: number;
maxChunkSize: number;
minChunkSize: number;
autoChunkSize: number;
chunkSize: number;
showChunkInput: boolean;
showPromptInput: boolean;
charsPointsPrice: number;
priceTip: string;
uploadRate: number;
chunkSizeField?: ChunkSizeFieldType;
};
type DatasetImportContextType = {
importSource: ImportDataSourceEnum;
parentId: string | undefined;
activeStep: number;
goToNext: () => void;
processParamsForm: UseFormReturn<ImportFormType, any>;
sources: ImportSourceItemType[];
setSources: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
} & TrainingFiledType;
type ChunkSizeFieldType = 'embeddingChunkSize' | 'qaChunkSize';
export type ImportFormType = {
mode: TrainingModeEnum;
way: ImportProcessWayEnum;
embeddingChunkSize: number;
qaChunkSize: number;
customSplitChar: string;
qaPrompt: string;
webSelector: string;
};
export const DatasetImportContext = createContext<DatasetImportContextType>({
importSource: ImportDataSourceEnum.fileLocal,
goToNext: function (): void {
throw new Error('Function not implemented.');
},
activeStep: 0,
parentId: undefined,
maxChunkSize: 0,
minChunkSize: 0,
showChunkInput: false,
showPromptInput: false,
sources: [],
setSources: function (value: SetStateAction<ImportSourceItemType[]>): void {
throw new Error('Function not implemented.');
},
chunkSize: 0,
chunkOverlapRatio: 0,
uploadRate: 0,
//@ts-ignore
processParamsForm: undefined,
autoChunkSize: 0,
charsPointsPrice: 0,
priceTip: ''
});
const DatasetImportContextProvider = ({ children }: { children: React.ReactNode }) => {
const { t } = useTranslation();
const router = useRouter();
const { source = ImportDataSourceEnum.fileLocal, parentId } = (router.query || {}) as {
source: ImportDataSourceEnum;
parentId?: string;
};
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
// step
const modeSteps: Record<ImportDataSourceEnum, { title: string }[]> = {
[ImportDataSourceEnum.reTraining]: [
{ title: t('dataset:core.dataset.import.Adjust parameters') },
{ title: t('common:core.dataset.import.Upload data') }
],
[ImportDataSourceEnum.fileLocal]: [
{
title: t('common:core.dataset.import.Select file')
},
{
title: t('common:core.dataset.import.Data Preprocessing')
},
{
title: t('common:core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.fileLink]: [
{
title: t('common:core.dataset.import.Select file')
},
{
title: t('common:core.dataset.import.Data Preprocessing')
},
{
title: t('common:core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.fileCustom]: [
{
title: t('common:core.dataset.import.Select file')
},
{
title: t('common:core.dataset.import.Data Preprocessing')
},
{
title: t('common:core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.csvTable]: [
{
title: t('common:core.dataset.import.Select file')
},
{
title: t('common:core.dataset.import.Data Preprocessing')
},
{
title: t('common:core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.externalFile]: [
{
title: t('common:core.dataset.import.Select file')
},
{
title: t('common:core.dataset.import.Data Preprocessing')
},
{
title: t('common:core.dataset.import.Upload data')
}
],
[ImportDataSourceEnum.apiDataset]: [
{
title: t('common:core.dataset.import.Select file')
},
{
title: t('common:core.dataset.import.Data Preprocessing')
},
{
title: t('common:core.dataset.import.Upload data')
}
]
};
const steps = modeSteps[source];
const { activeStep, goToNext, goToPrevious, MyStep } = useMyStep({
defaultStep: 0,
steps
});
const vectorModel = datasetDetail.vectorModel;
const agentModel = datasetDetail.agentModel;
const processParamsForm = useForm<ImportFormType>({
defaultValues: {
mode: TrainingModeEnum.chunk,
way: ImportProcessWayEnum.auto,
embeddingChunkSize: vectorModel?.defaultToken || 512,
qaChunkSize: Math.min(agentModel.maxResponse * 1, agentModel.maxContext * 0.7),
customSplitChar: '',
qaPrompt: Prompt_AgentQA.description,
webSelector: ''
}
});
const [sources, setSources] = useState<ImportSourceItemType[]>([]);
// watch form
const mode = processParamsForm.watch('mode');
const way = processParamsForm.watch('way');
const embeddingChunkSize = processParamsForm.watch('embeddingChunkSize');
const qaChunkSize = processParamsForm.watch('qaChunkSize');
const customSplitChar = processParamsForm.watch('customSplitChar');
const modeStaticParams: Record<TrainingModeEnum, TrainingFiledType> = {
[TrainingModeEnum.auto]: {
chunkOverlapRatio: 0.2,
maxChunkSize: 2048,
minChunkSize: 100,
autoChunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024,
chunkSize: vectorModel?.defaultToken ? vectorModel?.defaultToken * 2 : 1024,
showChunkInput: false,
showPromptInput: false,
charsPointsPrice: agentModel.charsPointsPrice || 0,
priceTip: t('dataset:import.Auto mode Estimated Price Tips', {
price: agentModel.charsPointsPrice
}),
uploadRate: 100
},
[TrainingModeEnum.chunk]: {
chunkSizeField: 'embeddingChunkSize' as ChunkSizeFieldType,
chunkOverlapRatio: 0.2,
maxChunkSize: vectorModel?.maxToken || 512,
minChunkSize: 100,
autoChunkSize: vectorModel?.defaultToken || 512,
chunkSize: embeddingChunkSize,
showChunkInput: true,
showPromptInput: false,
charsPointsPrice: vectorModel.charsPointsPrice || 0,
priceTip: t('dataset:import.Embedding Estimated Price Tips', {
price: vectorModel.charsPointsPrice
}),
uploadRate: 150
},
[TrainingModeEnum.qa]: {
chunkSizeField: 'qaChunkSize' as ChunkSizeFieldType,
chunkOverlapRatio: 0,
maxChunkSize: Math.min(agentModel.maxResponse * 4, agentModel.maxContext * 0.7),
minChunkSize: 4000,
autoChunkSize: Math.min(agentModel.maxResponse * 1, agentModel.maxContext * 0.7),
chunkSize: qaChunkSize,
showChunkInput: true,
showPromptInput: true,
charsPointsPrice: agentModel.charsPointsPrice || 0,
priceTip: t('dataset:import.Auto mode Estimated Price Tips', {
price: agentModel.charsPointsPrice
}),
uploadRate: 30
}
};
const selectModelStaticParam = modeStaticParams[mode];
const wayStaticPrams = {
[ImportProcessWayEnum.auto]: {
chunkSize: selectModelStaticParam.autoChunkSize,
customSplitChar: ''
},
[ImportProcessWayEnum.custom]: {
chunkSize: modeStaticParams[mode].chunkSize,
customSplitChar
}
};
const chunkSize = wayStaticPrams[way].chunkSize;
const contextValue = {
importSource: source,
parentId,
activeStep,
goToNext,
processParamsForm,
...selectModelStaticParam,
sources,
setSources,
chunkSize
};
return (
<DatasetImportContext.Provider value={contextValue}>
<Flex>
{activeStep === 0 ? (
<Flex alignItems={'center'}>
<IconButton
icon={<MyIcon name={'common/backFill'} w={'14px'} />}
aria-label={''}
size={'smSquare'}
borderRadius={'50%'}
variant={'whiteBase'}
mr={2}
onClick={() =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.collectionCard
}
})
}
/>
{t('common:common.Exit')}
</Flex>
) : (
<Button
variant={'whiteBase'}
leftIcon={<MyIcon name={'common/backFill'} w={'14px'} />}
onClick={goToPrevious}
>
{t('common:common.Last Step')}
</Button>
)}
<Box flex={1} />
</Flex>
{/* step */}
<Box
mt={4}
mb={5}
px={3}
py={[2, 4]}
bg={'myGray.50'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
borderRadius={'md'}
>
<Box maxW={['100%', '900px']} mx={'auto'}>
<MyStep />
</Box>
</Box>
{children}
</DatasetImportContext.Provider>
);
};
export default DatasetImportContextProvider;

319
projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx Normal file
View File

@@ -0,0 +1,319 @@
import React, { useCallback, useMemo, useRef } from 'react';
import {
Box,
Flex,
Input,
Button,
ModalBody,
ModalFooter,
Textarea,
useDisclosure
} from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
import Preview from '../components/Preview';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
import { useToast } from '@fastgpt/web/hooks/useToast';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
function DataProcess({ showPreviewChunks = true }: { showPreviewChunks: boolean }) {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const {
goToNext,
processParamsForm,
chunkSizeField,
minChunkSize,
showChunkInput,
showPromptInput,
maxChunkSize,
priceTip,
chunkSize
} = useContextSelector(DatasetImportContext, (v) => v);
const { getValues, setValue, register, watch } = processParamsForm;
const { toast } = useToast();
const mode = watch('mode');
const way = watch('way');
const {
isOpen: isOpenCustomPrompt,
onOpen: onOpenCustomPrompt,
onClose: onCloseCustomPrompt
} = useDisclosure();
const trainingModeList = useMemo(() => {
const list = Object.entries(TrainingTypeMap);
return list;
}, []);
const onSelectTrainWay = useCallback(
(e: TrainingModeEnum) => {
if (!feConfigs?.isPlus && !TrainingTypeMap[e]?.openSource) {
return toast({
status: 'warning',
title: t('common:common.system.Commercial version function')
});
}
setValue('mode', e);
},
[feConfigs?.isPlus, setValue, t, toast]
);
return (
<Box h={'100%'} display={['block', 'flex']} fontSize={'sm'}>
<Box
flex={'1 0 0'}
minW={['auto', '500px']}
maxW={'600px'}
h={['auto', '100%']}
overflow={'auto'}
pr={[0, 3]}
>
<Flex alignItems={'center'}>
<MyIcon name={'common/settingLight'} w={'20px'} />
<Box fontSize={'md'}>{t('dataset:data_process_setting')}</Box>
</Flex>
<Box display={['block', 'flex']} mt={4} alignItems={'center'}>
<FormLabel flex={'0 0 100px'}>{t('dataset:training_mode')}</FormLabel>
<LeftRadio
list={trainingModeList.map(([key, value]) => ({
title: t(value.label as any),
value: key,
tooltip: t(value.tooltip as any)
}))}
px={3}
py={2}
value={mode}
onChange={onSelectTrainWay}
defaultBg="white"
activeBg="white"
display={'flex'}
flexWrap={'wrap'}
/>
</Box>
<Box display={['block', 'flex']} mt={5}>
<FormLabel flex={'0 0 100px'}>{t('dataset:data_process_params')}</FormLabel>
<LeftRadio
list={[
{
title: t('common:core.dataset.import.Auto process'),
desc: t('common:core.dataset.import.Auto process desc'),
value: ImportProcessWayEnum.auto
},
{
title: t('dataset:custom_data_process_params'),
desc: t('dataset:custom_data_process_params_desc'),
value: ImportProcessWayEnum.custom,
children: way === ImportProcessWayEnum.custom && (
<Box mt={5}>
{showChunkInput && chunkSizeField && (
<Box>
<Flex alignItems={'center'}>
<Box>{t('dataset:ideal_chunk_length')}</Box>
<QuestionTip label={t('dataset:ideal_chunk_length_tips')} />
</Flex>
<Box
mt={1}
css={{
'& > span': {
display: 'block'
}
}}
>
<MyTooltip
label={t('common:core.dataset.import.Chunk Range', {
min: minChunkSize,
max: maxChunkSize
})}
>
<MyNumberInput
name={chunkSizeField}
min={minChunkSize}
max={maxChunkSize}
size={'sm'}
step={100}
value={chunkSize}
onChange={(e) => {
if (e === undefined) return;
setValue(chunkSizeField, +e);
}}
/>
</MyTooltip>
</Box>
</Box>
)}
<Box mt={3}>
<Box>
{t('common:core.dataset.import.Custom split char')}
<QuestionTip
label={t('common:core.dataset.import.Custom split char Tips')}
/>
</Box>
<Box mt={1}>
<Input
size={'sm'}
bg={'myGray.50'}
defaultValue={''}
placeholder="\n;======;==SPLIT=="
{...register('customSplitChar')}
/>
</Box>
</Box>
{showPromptInput && (
<Box mt={3}>
<Box>{t('common:core.dataset.collection.QA Prompt')}</Box>
<Box
position={'relative'}
py={2}
px={3}
bg={'myGray.50'}
fontSize={'xs'}
whiteSpace={'pre-wrap'}
border={'1px'}
borderColor={'borderColor.base'}
borderRadius={'md'}
maxH={'140px'}
overflow={'auto'}
_hover={{
'& .mask': {
display: 'block'
}
}}
>
{getValues('qaPrompt')}
<Box
display={'none'}
className="mask"
position={'absolute'}
top={0}
right={0}
bottom={0}
left={0}
background={
'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
}
>
<Button
size="xs"
variant={'whiteBase'}
leftIcon={<MyIcon name={'edit'} w={'13px'} />}
color={'black'}
position={'absolute'}
right={2}
bottom={2}
onClick={onOpenCustomPrompt}
>
{t('common:core.dataset.import.Custom prompt')}
</Button>
</Box>
</Box>
</Box>
)}
</Box>
)
}
]}
px={3}
py={3}
defaultBg="white"
activeBg="white"
value={way}
w={'100%'}
onChange={(e) => {
setValue('way', e);
}}
></LeftRadio>
</Box>
{feConfigs?.show_pay && (
<Box mt={5} pl={[0, '100px']} gap={3}>
<MyTag colorSchema={'gray'} py={1.5} borderRadius={'md'} px={3} whiteSpace={'wrap'}>
{priceTip}
</MyTag>
</Box>
)}
<Flex mt={5} gap={3} justifyContent={'flex-end'}>
<Button
onClick={() => {
goToNext();
}}
>
{t('common:common.Next Step')}
</Button>
</Flex>
</Box>
<Box flex={'1 0 0'} w={['auto', '0']} h={['auto', '100%']} pl={[0, 3]}>
<Preview showPreviewChunks={showPreviewChunks} />
</Box>
{isOpenCustomPrompt && (
<PromptTextarea
defaultValue={getValues('qaPrompt')}
onChange={(e) => {
setValue('qaPrompt', e);
}}
onClose={onCloseCustomPrompt}
/>
)}
</Box>
);
}
export default React.memo(DataProcess);
const PromptTextarea = ({
defaultValue,
onChange,
onClose
}: {
defaultValue: string;
onChange: (e: string) => void;
onClose: () => void;
}) => {
const ref = useRef<HTMLTextAreaElement>(null);
const { t } = useTranslation();
return (
<MyModal
isOpen
title={t('common:core.dataset.import.Custom prompt')}
iconSrc="modal/edit"
w={'600px'}
onClose={onClose}
>
<ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
<Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
<Box>{Prompt_AgentQA.fixedText}</Box>
</ModalBody>
<ModalFooter>
<Button
onClick={() => {
const val = ref.current?.value || Prompt_AgentQA.description;
onChange(val);
onClose();
}}
>
{t('common:common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};

24
projects/app/src/pageComponents/dataset/detail/Import/commonProgress/PreviewData.tsx Normal file
View File

@@ -0,0 +1,24 @@
import React from 'react';
import Preview from '../components/Preview';
import { Box, Button, Flex } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const PreviewData = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => {
const { t } = useTranslation();
const goToNext = useContextSelector(DatasetImportContext, (v) => v.goToNext);
return (
<Flex flexDirection={'column'} h={'100%'}>
<Box flex={'1 0 0 '}>
<Preview showPreviewChunks={showPreviewChunks} />
</Box>
<Flex mt={2} justifyContent={'flex-end'}>
<Button onClick={goToNext}>{t('common:common.Next Step')}</Button>
</Flex>
</Flex>
);
};
export default React.memo(PreviewData);

285
projects/app/src/pageComponents/dataset/detail/Import/commonProgress/Upload.tsx Normal file
View File

@@ -0,0 +1,285 @@
import React, { useMemo, useRef } from 'react';
import { QuestionOutlineIcon } from '@chakra-ui/icons';
import {
Box,
TableContainer,
Table,
Thead,
Tr,
Th,
Td,
Tbody,
Flex,
Button,
IconButton,
Tooltip
} from '@chakra-ui/react';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { useRouter } from 'next/router';
import { TabEnum } from '../../../../../pages/dataset/detail/index';
import {
postCreateDatasetApiDatasetCollection,
postCreateDatasetCsvTableCollection,
postCreateDatasetExternalFileCollection,
postCreateDatasetFileCollection,
postCreateDatasetLinkCollection,
postCreateDatasetTextCollection,
postReTrainingDatasetFileCollection
} from '@/web/core/dataset/api';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { DatasetImportContext, type ImportFormType } from '../Context';
const Upload = () => {
const { t } = useTranslation();
const { toast } = useToast();
const router = useRouter();
const { collectionId = '' } = router.query as {
collectionId: string;
};
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const retrainNewCollectionId = useRef('');
const { importSource, parentId, sources, setSources, processParamsForm, chunkSize } =
useContextSelector(DatasetImportContext, (v) => v);
const { handleSubmit } = processParamsForm;
const { totalFilesCount, waitingFilesCount, allFinished, hasCreatingFiles } = useMemo(() => {
const totalFilesCount = sources.length;
const { waitingFilesCount, allFinished, hasCreatingFiles } = sources.reduce(
(acc, file) => {
if (file.createStatus === 'waiting') acc.waitingFilesCount++;
if (file.createStatus === 'creating') acc.hasCreatingFiles = true;
if (file.createStatus !== 'finish') acc.allFinished = false;
return acc;
},
{ waitingFilesCount: 0, allFinished: true, hasCreatingFiles: false }
);
return { totalFilesCount, waitingFilesCount, allFinished, hasCreatingFiles };
}, [sources]);
const buttonText = useMemo(() => {
if (waitingFilesCount === totalFilesCount) {
return t('common:core.dataset.import.Start upload');
} else if (allFinished) {
return t('common:core.dataset.import.Upload complete');
} else {
return t('common:core.dataset.import.Continue upload');
}
}, [waitingFilesCount, totalFilesCount, allFinished, t]);
const { runAsync: startUpload, loading: isLoading } = useRequest2(
async ({ mode, customSplitChar, qaPrompt, webSelector }: ImportFormType) => {
if (sources.length === 0) return;
const filterWaitingSources = sources.filter((item) => item.createStatus === 'waiting');
// Batch create collection and upload chunks
for await (const item of filterWaitingSources) {
setSources((state) =>
state.map((source) =>
source.id === item.id
? {
...source,
createStatus: 'creating'
}
: source
)
);
// create collection
const commonParams = {
parentId,
trainingType: mode,
datasetId: datasetDetail._id,
chunkSize,
chunkSplitter: customSplitChar,
qaPrompt,
name: item.sourceName
};
if (importSource === ImportDataSourceEnum.reTraining) {
const res = await postReTrainingDatasetFileCollection({
...commonParams,
collectionId
});
retrainNewCollectionId.current = res.collectionId;
} else if (importSource === ImportDataSourceEnum.fileLocal && item.dbFileId) {
await postCreateDatasetFileCollection({
...commonParams,
fileId: item.dbFileId
});
} else if (importSource === ImportDataSourceEnum.fileLink && item.link) {
await postCreateDatasetLinkCollection({
...commonParams,
link: item.link,
metadata: {
webPageSelector: webSelector
}
});
} else if (importSource === ImportDataSourceEnum.fileCustom && item.rawText) {
// manual collection
await postCreateDatasetTextCollection({
...commonParams,
text: item.rawText
});
} else if (importSource === ImportDataSourceEnum.csvTable && item.dbFileId) {
await postCreateDatasetCsvTableCollection({
...commonParams,
fileId: item.dbFileId
});
} else if (importSource === ImportDataSourceEnum.externalFile && item.externalFileUrl) {
await postCreateDatasetExternalFileCollection({
...commonParams,
externalFileUrl: item.externalFileUrl,
externalFileId: item.externalFileId,
filename: item.sourceName
});
} else if (importSource === ImportDataSourceEnum.apiDataset && item.apiFileId) {
await postCreateDatasetApiDatasetCollection({
...commonParams,
apiFileId: item.apiFileId
});
}
setSources((state) =>
state.map((source) =>
source.id === item.id
? {
...source,
createStatus: 'finish'
}
: source
)
);
}
},
{
onSuccess() {
if (!sources.some((file) => file.errorMsg !== undefined)) {
toast({
title:
importSource === ImportDataSourceEnum.reTraining
? t('dataset:retrain_task_submitted')
: t('common:core.dataset.import.Import success'),
status: 'success'
});
}
// Close import page
router.replace({
query: {
datasetId: datasetDetail._id,
currentTab: TabEnum.collectionCard
}
});
},
onError(error) {
setSources((state) =>
state.map((source) =>
source.createStatus === 'creating'
? {
...source,
createStatus: 'waiting',
errorMsg: error.message || t('file:upload_failed')
}
: source
)
);
},
errorToast: t('file:upload_failed')
}
);
return (
<Box h={'100%'} overflow={'auto'}>
<TableContainer>
<Table variant={'simple'} fontSize={'sm'} draggable={false}>
<Thead draggable={false}>
<Tr bg={'myGray.100'} mb={2}>
<Th borderLeftRadius={'md'} overflow={'hidden'} borderBottom={'none'} py={4}>
{t('common:core.dataset.import.Source name')}
</Th>
<Th borderBottom={'none'} py={4}>
{t('common:core.dataset.import.Upload status')}
</Th>
<Th borderRightRadius={'md'} borderBottom={'none'} py={4}>
{t('common:common.Action')}
</Th>
</Tr>
</Thead>
<Tbody>
{sources.map((item) => (
<Tr key={item.id}>
<Td>
<Flex alignItems={'center'}>
<MyIcon name={item.icon as any} w={'16px'} mr={1} />
<Box whiteSpace={'wrap'} maxW={'30vw'}>
{item.sourceName}
</Box>
</Flex>
</Td>
<Td>
<Box display={'inline-block'}>
{item.errorMsg ? (
<Tooltip label={item.errorMsg} fontSize="md">
<Flex alignItems="center">
<MyTag colorSchema={'red'}>{t('common:common.Error')}</MyTag>
<QuestionOutlineIcon ml={2} color="red.500" w="14px" />
</Flex>
</Tooltip>
) : (
<>
{item.createStatus === 'waiting' && (
<MyTag colorSchema={'gray'}>{t('common:common.Waiting')}</MyTag>
)}
{item.createStatus === 'creating' && (
<MyTag colorSchema={'blue'}>{t('common:common.Creating')}</MyTag>
)}
{item.createStatus === 'finish' && (
<MyTag colorSchema={'green'}>{t('common:common.Finish')}</MyTag>
)}
</>
)}
</Box>
</Td>
<Td>
{!hasCreatingFiles && item.createStatus !== 'finish' && (
<IconButton
variant={'grayDanger'}
size={'sm'}
icon={<MyIcon name={'delete'} w={'14px'} />}
aria-label={'Delete file'}
onClick={() => {
setSources((prevFiles) => prevFiles.filter((file) => file.id !== item.id));
}}
/>
)}
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
<Flex justifyContent={'flex-end'} mt={4}>
<Button isLoading={isLoading} onClick={handleSubmit((data) => startUpload(data))}>
{totalFilesCount > 0 &&
`${t('common:core.dataset.import.Total files', {
total: totalFilesCount
})} | `}
{buttonText}
</Button>
</Flex>
</Box>
);
};
export default Upload;

336
projects/app/src/pageComponents/dataset/detail/Import/components/FileSelector.tsx Normal file
View File

@@ -0,0 +1,336 @@
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { Box, FlexProps } from '@chakra-ui/react';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import React, { DragEvent, useCallback, useMemo, useState } from 'react';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { uploadFile2DB } from '@/web/common/file/controller';
import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import { useI18n } from '@/web/context/I18n';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
export type SelectFileItemType = {
fileId: string;
folderPath: string;
file: File;
};
const FileSelector = ({
fileType,
selectFiles,
setSelectFiles,
onStartSelect,
onFinishSelect,
...props
}: {
fileType: string;
selectFiles: ImportSourceItemType[];
setSelectFiles: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
onStartSelect: () => void;
onFinishSelect: () => void;
} & FlexProps) => {
const { t } = useTranslation();
const { fileT } = useI18n();
const { toast } = useToast();
const { feConfigs } = useSystemStore();
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
const maxCount = feConfigs?.uploadFileMaxAmount || 1000;
const maxSize = (feConfigs?.uploadFileMaxSize || 1024) * 1024 * 1024;
const { File, onOpen } = useSelectFile({
fileType,
multiple: true,
maxCount
});
const [isDragging, setIsDragging] = useState(false);
const isMaxSelected = useMemo(
() => selectFiles.length >= maxCount,
[maxCount, selectFiles.length]
);
const filterTypeReg = new RegExp(
`(${fileType
.split(',')
.map((item) => item.trim())
.join('|')})$`,
'i'
);
const { mutate: onSelectFile, isLoading } = useRequest({
mutationFn: async (files: SelectFileItemType[]) => {
{
onStartSelect();
setSelectFiles((state) => {
const formatFiles = files.map<ImportSourceItemType>((selectFile) => {
const { fileId, file } = selectFile;
return {
id: fileId,
createStatus: 'waiting',
file,
sourceName: file.name,
sourceSize: formatFileSize(file.size),
icon: getFileIcon(file.name),
isUploading: true,
uploadedFileRate: 0
};
});
const results = formatFiles.concat(state).slice(0, maxCount);
return results;
});
try {
// upload file
await Promise.all(
files.map(async ({ fileId, file }) => {
const { fileId: uploadFileId } = await uploadFile2DB({
file,
bucketName: BucketNameEnum.dataset,
data: {
datasetId
},
percentListen: (e) => {
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
uploadedFileRate: item.uploadedFileRate
? Math.max(e, item.uploadedFileRate)
: e
}
: item
)
);
}
});
setSelectFiles((state) =>
state.map((item) =>
item.id === fileId
? {
...item,
dbFileId: uploadFileId,
isUploading: false,
uploadedFileRate: 100
}
: item
)
);
})
);
} catch (error) {
console.log(error);
}
onFinishSelect();
}
}
});
const selectFileCallback = useCallback(
(files: SelectFileItemType[]) => {
if (selectFiles.length + files.length > maxCount) {
files = files.slice(0, maxCount - selectFiles.length);
toast({
status: 'warning',
title: fileT('some_file_count_exceeds_limit', { maxCount })
});
}
// size check
if (!maxSize) {
return onSelectFile(files);
}
const filterFiles = files.filter((item) => item.file.size <= maxSize);
if (filterFiles.length < files.length) {
toast({
status: 'warning',
title: fileT('some_file_size_exceeds_limit', { maxSize: formatFileSize(maxSize) })
});
}
return onSelectFile(filterFiles);
},
[fileT, maxCount, maxSize, onSelectFile, selectFiles.length, toast]
);
const handleDragEnter = (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(true);
};
const handleDragLeave = (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(false);
};
const handleDrop = async (e: DragEvent<HTMLDivElement>) => {
e.preventDefault();
setIsDragging(false);
const items = e.dataTransfer.items;
const fileList: SelectFileItemType[] = [];
const firstEntry = items[0].webkitGetAsEntry();
if (firstEntry?.isDirectory && items.length === 1) {
{
const readFile = (entry: any) => {
return new Promise((resolve) => {
entry.file((file: File) => {
const folderPath = (entry.fullPath || '').split('/').slice(2, -1).join('/');
if (filterTypeReg.test(file.name)) {
fileList.push({
fileId: getNanoid(),
folderPath,
file
});
}
resolve(file);
});
});
};
const traverseFileTree = (dirReader: any) => {
return new Promise((resolve) => {
let fileNum = 0;
dirReader.readEntries(async (entries: any[]) => {
for await (const entry of entries) {
if (entry.isFile) {
await readFile(entry);
fileNum++;
} else if (entry.isDirectory) {
await traverseFileTree(entry.createReader());
}
}
// chrome: readEntries will return 100 entries at most
if (fileNum === 100) {
await traverseFileTree(dirReader);
}
resolve('');
});
});
};
for await (const item of items) {
const entry = item.webkitGetAsEntry();
if (entry) {
if (entry.isFile) {
await readFile(entry);
} else if (entry.isDirectory) {
//@ts-ignore
await traverseFileTree(entry.createReader());
}
}
}
}
} else if (firstEntry?.isFile) {
const files = Array.from(e.dataTransfer.files);
let isErr = files.some((item) => item.type === '');
if (isErr) {
return toast({
title: t('file:upload_error_description'),
status: 'error'
});
}
fileList.push(
...files
.filter((item) => filterTypeReg.test(item.name))
.map((file) => ({
fileId: getNanoid(),
folderPath: '',
file
}))
);
} else {
return toast({
title: fileT('upload_error_description'),
status: 'error'
});
}
selectFileCallback(fileList.slice(0, maxCount));
};
return (
<MyBox
isLoading={isLoading}
display={'flex'}
flexDirection={'column'}
alignItems={'center'}
justifyContent={'center'}
px={3}
py={[4, 7]}
borderWidth={'1.5px'}
borderStyle={'dashed'}
borderRadius={'md'}
{...(isMaxSelected
? {}
: {
cursor: 'pointer',
_hover: {
bg: 'primary.50',
borderColor: 'primary.600'
},
borderColor: isDragging ? 'primary.600' : 'borderColor.high',
onDragEnter: handleDragEnter,
onDragOver: (e) => e.preventDefault(),
onDragLeave: handleDragLeave,
onDrop: handleDrop,
onClick: onOpen
})}
{...props}
>
<MyIcon name={'common/uploadFileFill'} w={'32px'} />
{isMaxSelected ? (
<>
<Box color={'myGray.500'} fontSize={'xs'}>
{t('file:reached_max_file_count')}
</Box>
</>
) : (
<>
<Box fontWeight={'bold'}>
{isDragging
? fileT('release_the_mouse_to_upload_the_file')
: fileT('select_and_drag_file_tip')}
</Box>
{/* file type */}
<Box color={'myGray.500'} fontSize={'xs'}>
{fileT('support_file_type', { fileType })}
</Box>
<Box color={'myGray.500'} fontSize={'xs'}>
{/* max count */}
{maxCount && fileT('support_max_count', { maxCount })}
{/* max size */}
{maxSize && fileT('support_max_size', { maxSize: formatFileSize(maxSize) })}
</Box>
<File
onSelect={(files) =>
selectFileCallback(
files.map((file) => ({
fileId: getNanoid(),
folderPath: '',
file
}))
)
}
/>
</>
)}
</MyBox>
);
};
export default React.memo(FileSelector);

65
projects/app/src/pageComponents/dataset/detail/Import/components/FileSourceSelector.tsx Normal file
View File

@@ -0,0 +1,65 @@
import React, { useState } from 'react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { ModalBody, ModalFooter, Button } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import { useRouter } from 'next/router';
import { TabEnum } from '../../../../../pages/dataset/detail';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
const FileModeSelector = ({ onClose }: { onClose: () => void }) => {
const { t } = useTranslation();
const router = useRouter();
const [value, setValue] = useState<ImportDataSourceEnum>(ImportDataSourceEnum.fileLocal);
return (
<MyModal
isOpen
onClose={onClose}
iconSrc="modal/selectSource"
title={t('common:core.dataset.import.Select source')}
w={'600px'}
>
<ModalBody px={6} py={4}>
<LeftRadio
list={[
{
title: t('common:core.dataset.import.Local file'),
desc: t('common:core.dataset.import.Local file desc'),
value: ImportDataSourceEnum.fileLocal
},
{
title: t('common:core.dataset.import.Web link'),
desc: t('common:core.dataset.import.Web link desc'),
value: ImportDataSourceEnum.fileLink
},
{
title: t('common:core.dataset.import.Custom text'),
desc: t('common:core.dataset.import.Custom text desc'),
value: ImportDataSourceEnum.fileCustom
}
]}
value={value}
onChange={setValue}
/>
</ModalBody>
<ModalFooter>
<Button
onClick={() =>
router.replace({
query: {
...router.query,
currentTab: TabEnum.import,
source: value
}
})
}
>
{t('common:common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};
export default FileModeSelector;

102
projects/app/src/pageComponents/dataset/detail/Import/components/Preview.tsx Normal file
View File

@@ -0,0 +1,102 @@
import React, { useState } from 'react';
import { Box, Flex, Grid, IconButton } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import MyMenu from '@fastgpt/web/components/common/MyMenu';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import dynamic from 'next/dynamic';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const PreviewRawText = dynamic(() => import('./PreviewRawText'));
const PreviewChunks = dynamic(() => import('./PreviewChunks'));
const Preview = ({ showPreviewChunks }: { showPreviewChunks: boolean }) => {
const { t } = useTranslation();
const { sources } = useContextSelector(DatasetImportContext, (v) => v);
const [previewRawTextSource, setPreviewRawTextSource] = useState<ImportSourceItemType>();
const [previewChunkSource, setPreviewChunkSource] = useState<ImportSourceItemType>();
return (
<Box h={'100%'} w={'100%'} display={['block', 'flex']} flexDirection={'column'}>
<Flex alignItems={'center'}>
<MyIcon name={'core/dataset/fileCollection'} w={'20px'} />
<Box fontSize={'md'}>{t('common:core.dataset.import.Sources list')}</Box>
</Flex>
<Box mt={3} flex={'1 0 0'} h={['auto', 0]} width={'100%'} overflowY={'auto'}>
<Grid w={'100%'} gap={3} gridTemplateColumns={['1fr', '1fr', '1fr', '1fr', '1fr 1fr']}>
{sources.map((source) => (
<Flex
key={source.id}
bg={'white'}
p={4}
borderRadius={'md'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
boxShadow={'2'}
alignItems={'center'}
>
<MyIcon name={source.icon as any} w={['1rem', '1.25rem']} />
<Box mx={1} flex={'1 0 0'} wordBreak={'break-all'} fontSize={'sm'}>
{source.sourceName}
</Box>
{showPreviewChunks && (
<Box fontSize={'xs'} color={'myGray.600'}>
<MyMenu
Button={
<IconButton
icon={<MyIcon name={'common/viewLight'} w={'14px'} p={2} />}
aria-label={''}
size={'sm'}
variant={'whitePrimary'}
/>
}
menuList={[
{
children: [
{
label: (
<Flex alignItems={'center'}>
<MyIcon name={'core/dataset/fileCollection'} w={'14px'} mr={2} />
{t('common:core.dataset.import.Preview raw text')}
</Flex>
),
onClick: () => setPreviewRawTextSource(source)
},
{
label: (
<Flex alignItems={'center'}>
<MyIcon name={'core/dataset/splitLight'} w={'14px'} mr={2} />
{t('common:core.dataset.import.Preview chunks')}
</Flex>
),
onClick: () => setPreviewChunkSource(source)
}
]
}
]}
/>
</Box>
)}
</Flex>
))}
</Grid>
</Box>
{!!previewRawTextSource && (
<PreviewRawText
previewSource={previewRawTextSource}
onClose={() => setPreviewRawTextSource(undefined)}
/>
)}
{!!previewChunkSource && (
<PreviewChunks
previewSource={previewChunkSource}
onClose={() => setPreviewChunkSource(undefined)}
/>
)}
</Box>
);
};
export default React.memo(Preview);

102
projects/app/src/pageComponents/dataset/detail/Import/components/PreviewChunks.tsx Normal file
View File

@@ -0,0 +1,102 @@
import React from 'react';
import { Box } from '@chakra-ui/react';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer';
import { getPreviewChunks } from '@/web/core/dataset/api';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { getPreviewSourceReadType } from '../utils';
const PreviewChunks = ({
previewSource,
onClose
}: {
previewSource: ImportSourceItemType;
onClose: () => void;
}) => {
const { importSource, chunkSize, chunkOverlapRatio, processParamsForm } = useContextSelector(
DatasetImportContext,
(v) => v
);
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
const { data = [], loading: isLoading } = useRequest2(
async () => {
if (importSource === ImportDataSourceEnum.fileCustom) {
const customSplitChar = processParamsForm.getValues('customSplitChar');
const { chunks } = splitText2Chunks({
text: previewSource.rawText || '',
chunkLen: chunkSize,
overlapRatio: chunkOverlapRatio,
customReg: customSplitChar ? [customSplitChar] : []
});
return chunks.map((chunk) => ({
q: chunk,
a: ''
}));
}
return getPreviewChunks({
datasetId,
type: getPreviewSourceReadType(previewSource),
sourceId:
previewSource.dbFileId ||
previewSource.link ||
previewSource.externalFileUrl ||
previewSource.apiFileId ||
'',
chunkSize,
overlapRatio: chunkOverlapRatio,
customSplitChar: processParamsForm.getValues('customSplitChar'),
selector: processParamsForm.getValues('webSelector'),
isQAImport: importSource === ImportDataSourceEnum.csvTable,
externalFileId: previewSource.externalFileId
});
},
{
manual: false
}
);
return (
<MyRightDrawer
onClose={onClose}
iconSrc={previewSource.icon}
title={previewSource.sourceName}
isLoading={isLoading}
maxW={['90vw', '40vw']}
px={0}
>
<Box overflowY={'auto'} px={5} fontSize={'sm'}>
{data.map((item, index) => (
<Box
key={index}
whiteSpace={'pre-wrap'}
fontSize={'sm'}
p={4}
bg={index % 2 === 0 ? 'white' : 'myWhite.600'}
mb={3}
borderRadius={'md'}
borderWidth={'1px'}
borderColor={'borderColor.low'}
boxShadow={'2'}
_notLast={{
mb: 2
}}
>
<Box color={'myGray.900'}>{item.q}</Box>
<Box color={'myGray.500'}>{item.a}</Box>
</Box>
))}
</Box>
</MyRightDrawer>
);
};
export default React.memo(PreviewChunks);

78
projects/app/src/pageComponents/dataset/detail/Import/components/PreviewRawText.tsx Normal file
View File

@@ -0,0 +1,78 @@
import React from 'react';
import { Box } from '@chakra-ui/react';
import { ImportSourceItemType } from '@/web/core/dataset/type';
import { getPreviewFileContent } from '@/web/common/file/api';
import MyRightDrawer from '@fastgpt/web/components/common/MyDrawer/MyRightDrawer';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { getPreviewSourceReadType } from '../utils';
const PreviewRawText = ({
previewSource,
onClose
}: {
previewSource: ImportSourceItemType;
onClose: () => void;
}) => {
const { toast } = useToast();
const { importSource, processParamsForm } = useContextSelector(DatasetImportContext, (v) => v);
const datasetId = useContextSelector(DatasetPageContext, (v) => v.datasetId);
const { data, loading: isLoading } = useRequest2(
async () => {
if (importSource === ImportDataSourceEnum.fileCustom && previewSource.rawText) {
return {
previewContent: previewSource.rawText.slice(0, 3000)
};
}
return getPreviewFileContent({
datasetId,
type: getPreviewSourceReadType(previewSource),
sourceId:
previewSource.dbFileId ||
previewSource.link ||
previewSource.externalFileUrl ||
previewSource.apiFileId ||
'',
isQAImport: importSource === ImportDataSourceEnum.csvTable,
selector: processParamsForm.getValues('webSelector'),
externalFileId: previewSource.externalFileId
});
},
{
refreshDeps: [previewSource.dbFileId, previewSource.link, previewSource.externalFileUrl],
manual: false,
onError(err) {
toast({
status: 'warning',
title: getErrText(err)
});
}
}
);
const rawText = data?.previewContent || '';
return (
<MyRightDrawer
onClose={onClose}
iconSrc={previewSource.icon}
title={previewSource.sourceName}
isLoading={isLoading}
px={0}
>
<Box whiteSpace={'pre-wrap'} overflowY={'auto'} px={5} fontSize={'sm'}>
{rawText}
</Box>
</MyRightDrawer>
);
};
export default React.memo(PreviewRawText);

123
projects/app/src/pageComponents/dataset/detail/Import/components/RenderFiles.tsx Normal file
View File

@@ -0,0 +1,123 @@
import React, { useState } from 'react';
import {
Flex,
TableContainer,
Table,
Thead,
Tr,
Th,
Td,
Tbody,
Progress,
IconButton
} from '@chakra-ui/react';
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import dynamic from 'next/dynamic';
import { useI18n } from '@/web/context/I18n';
const PreviewRawText = dynamic(() => import('./PreviewRawText'));
export const RenderUploadFiles = ({
files,
setFiles,
showPreviewContent
}: {
files: ImportSourceItemType[];
setFiles: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
showPreviewContent?: boolean;
}) => {
const { t } = useTranslation();
const { fileT } = useI18n();
const [previewFile, setPreviewFile] = useState<ImportSourceItemType>();
return files.length > 0 ? (
<>
<TableContainer mt={5}>
<Table variant={'simple'} fontSize={'sm'} draggable={false}>
<Thead draggable={false}>
<Tr bg={'myGray.100'} mb={2}>
<Th borderLeftRadius={'md'} borderBottom={'none'} py={4}>
{fileT('file_name')}
</Th>
<Th borderBottom={'none'} py={4}>
{t('common:core.dataset.import.Upload file progress')}
</Th>
<Th borderBottom={'none'} py={4}>
{fileT('file_size')}
</Th>
<Th borderRightRadius={'md'} borderBottom={'none'} py={4}>
{t('common:common.Action')}
</Th>
</Tr>
</Thead>
<Tbody>
{files.map((item) => (
<Tr key={item.id}>
<Td>
<Flex alignItems={'center'}>
<MyIcon name={item.icon as any} w={'16px'} mr={1} />
{item.sourceName}
</Flex>
</Td>
<Td>
<Flex alignItems={'center'} fontSize={'xs'}>
<Progress
value={item.uploadedFileRate}
h={'6px'}
w={'100%'}
maxW={'210px'}
size="sm"
borderRadius={'20px'}
colorScheme={(item.uploadedFileRate || 0) >= 100 ? 'green' : 'blue'}
bg="myGray.200"
hasStripe
isAnimated
mr={2}
/>
{`${item.uploadedFileRate}%`}
</Flex>
</Td>
<Td>{item.sourceSize}</Td>
<Td>
{!item.isUploading && (
<Flex alignItems={'center'} gap={4}>
{showPreviewContent && (
<MyTooltip label={t('common:core.dataset.import.Preview raw text')}>
<IconButton
variant={'whitePrimary'}
size={'sm'}
icon={<MyIcon name={'common/viewLight'} w={'18px'} />}
aria-label={''}
onClick={() => setPreviewFile(item)}
/>
</MyTooltip>
)}
<IconButton
variant={'grayDanger'}
size={'sm'}
icon={<MyIcon name={'delete'} w={'14px'} />}
aria-label={''}
onClick={() => {
setFiles((state) => state.filter((file) => file.id !== item.id));
}}
/>
</Flex>
)}
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
{!!previewFile && (
<PreviewRawText previewSource={previewFile} onClose={() => setPreviewFile(undefined)} />
)}
</>
) : null;
};
export default RenderUploadFiles;

283
projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx Normal file
View File

@@ -0,0 +1,283 @@
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
import React, { useCallback, useMemo, useState } from 'react';
import dynamic from 'next/dynamic';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { Box, Button, Checkbox, Flex } from '@chakra-ui/react';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { getApiDatasetFileList, getApiDatasetFileListExistId } from '@/web/core/dataset/api';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
import FolderPath from '@/components/common/folder/Path';
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
import { useMount } from 'ahooks';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const APIDatasetCollection = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <CustomAPIFileInput />}
{activeStep === 1 && <DataProcess showPreviewChunks={true} />}
{activeStep === 2 && <Upload />}
</>
);
};
export default React.memo(APIDatasetCollection);
const CustomAPIFileInput = () => {
const { t } = useTranslation();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const goToNext = useContextSelector(DatasetImportContext, (v) => v.goToNext);
const sources = useContextSelector(DatasetImportContext, (v) => v.sources);
const setSources = useContextSelector(DatasetImportContext, (v) => v.setSources);
const [selectFiles, setSelectFiles] = useState<APIFileItem[]>([]);
const [parent, setParent] = useState<ParentTreePathItemType>({
parentId: '',
parentName: ''
});
const [paths, setPaths] = useState<ParentTreePathItemType[]>([]);
const [searchKey, setSearchKey] = useState('');
const { data: fileList = [], loading } = useRequest2(
async () => {
return getApiDatasetFileList({
datasetId: datasetDetail._id,
parentId: parent?.parentId,
searchKey: searchKey
});
},
{
refreshDeps: [datasetDetail._id, datasetDetail.apiServer, parent, searchKey],
throttleWait: 500,
manual: false
}
);
const { data: existIdList = [] } = useRequest2(
() => getApiDatasetFileListExistId({ datasetId: datasetDetail._id }),
{
manual: false
}
);
// Init selected files
useMount(() => {
setSelectFiles(sources.map((item) => item.apiFile).filter(Boolean) as APIFileItem[]);
});
const { runAsync: onclickNext, loading: onNextLoading } = useRequest2(
async () => {
// Computed all selected files
const getFilesRecursively = async (files: APIFileItem[]): Promise<APIFileItem[]> => {
const allFiles: APIFileItem[] = [];
for (const file of files) {
if (file.type === 'folder') {
const folderFiles = await getApiDatasetFileList({
datasetId: datasetDetail._id,
parentId: file?.id
});
const subFiles = await getFilesRecursively(folderFiles);
allFiles.push(...subFiles);
} else {
allFiles.push(file);
}
}
return allFiles;
};
const allFiles = await getFilesRecursively(selectFiles);
setSources(
allFiles
.filter((item) => !existIdList.includes(item.id))
.map((item) => ({
id: item.id,
apiFileId: item.id,
apiFile: item,
createStatus: 'waiting',
sourceName: item.name,
icon: getSourceNameIcon({ sourceName: item.name }) as any
}))
);
},
{
onSuccess() {
goToNext();
}
}
);
const handleItemClick = useCallback(
(item: APIFileItem) => {
if (item.hasChild) {
setPaths((state) => [...state, { parentId: item.id, parentName: item.name }]);
return setParent({
parentId: item.id,
parentName: item.name
});
}
const isCurrentlySelected = selectFiles.some((file) => file.id === item.id);
if (isCurrentlySelected) {
setSelectFiles((state) => state.filter((file) => file.id !== item.id));
} else {
setSelectFiles((state) => [...state, item]);
}
},
[selectFiles]
);
const handleSelectAll = useCallback(() => {
const isAllSelected = fileList.length === selectFiles.length;
if (isAllSelected) {
setSelectFiles([]);
} else {
setSelectFiles(fileList);
}
}, [fileList, selectFiles]);
return (
<MyBox isLoading={loading} position="relative" h="full">
<Flex flexDirection={'column'} h="full">
<Flex justifyContent={'space-between'}>
<FolderPath
paths={paths}
onClick={(parentId) => {
const index = paths.findIndex((item) => item.parentId === parentId);
setParent(paths[index]);
setPaths(paths.slice(0, index + 1));
}}
/>
{datasetDetail.apiServer && (
<Box w={'240px'}>
<SearchInput
value={searchKey}
onChange={(e) => setSearchKey(e.target.value)}
placeholder={t('common:core.workflow.template.Search')}
/>
</Box>
)}
</Flex>
<Box flex={1} overflowY="auto" mb={16}>
<Box ml={2} mt={3}>
<Flex
alignItems={'center'}
py={3}
cursor={'pointer'}
bg={'myGray.50'}
pl={7}
rounded={'8px'}
fontSize={'sm'}
fontWeight={'medium'}
color={'myGray.900'}
onClick={(e) => {
if (!(e.target as HTMLElement).closest('.checkbox')) {
handleSelectAll();
}
}}
>
<Checkbox
className="checkbox"
mr={2}
isChecked={fileList.length === selectFiles.length}
onChange={handleSelectAll}
/>
{t('common:Select_all')}
</Flex>
{fileList.map((item) => {
const isFolder = item.type === 'folder';
const isExists = existIdList.includes(item.id);
const isChecked = isExists || selectFiles.some((file) => file.id === item.id);
return (
<Flex
key={item.id}
py={3}
_hover={{ bg: 'primary.50' }}
pl={7}
cursor={'pointer'}
onClick={(e) => {
if (isExists) return;
if (!(e.target as HTMLElement).closest('.checkbox')) {
handleItemClick(item);
}
}}
>
<Checkbox
className="checkbox"
mr={2.5}
isChecked={isChecked}
isDisabled={isExists}
onChange={(e) => {
e.stopPropagation();
if (isExists) return;
if (isChecked) {
setSelectFiles((state) => state.filter((file) => file.id !== item.id));
} else {
setSelectFiles((state) => [...state, item]);
}
}}
/>
<MyIcon
name={
!isFolder
? (getSourceNameIcon({ sourceName: item.name }) as any)
: 'common/folderFill'
}
w={'18px'}
mr={1.5}
/>
<Box fontSize={'sm'} fontWeight={'medium'} color={'myGray.900'}>
{item.name}
</Box>
{item.hasChild && <MyIcon name="core/chat/chevronRight" w={'18px'} ml={2} />}
</Flex>
);
})}
</Box>
</Box>
<Box
position="absolute"
display={'flex'}
justifyContent={'end'}
bottom={0}
left={0}
right={0}
p={4}
>
<Button
isDisabled={selectFiles.length === 0}
isLoading={onNextLoading}
onClick={onclickNext}
>
{selectFiles.length > 0
? `${t('common:core.dataset.import.Total files', { total: selectFiles.length })} | `
: ''}
{t('common:common.Next Step')}
</Button>
</Box>
</Flex>
</MyBox>
);
};

186
projects/app/src/pageComponents/dataset/detail/Import/diffSource/ExternalFile.tsx Normal file
View File

@@ -0,0 +1,186 @@
import React, { useEffect } from 'react';
import dynamic from 'next/dynamic';
import { useTranslation } from 'next-i18next';
import { useFieldArray, useForm } from 'react-hook-form';
import {
Box,
Button,
Flex,
Table,
Thead,
Tbody,
Tr,
Th,
Td,
TableContainer,
Input
} from '@chakra-ui/react';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import MyIcon from '@fastgpt/web/components/common/Icon';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
import { getFileIcon } from '@fastgpt/global/common/file/icon';
import { SmallAddIcon } from '@chakra-ui/icons';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const ExternalFileCollection = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <CustomLinkInput />}
{activeStep === 1 && <DataProcess showPreviewChunks={true} />}
{activeStep === 2 && <Upload />}
</>
);
};
export default React.memo(ExternalFileCollection);
const CustomLinkInput = () => {
const { t } = useTranslation();
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
const { register, reset, handleSubmit, control } = useForm<{
list: {
sourceName: string;
externalFileUrl: string;
externalFileId: string;
}[];
}>({
defaultValues: {
list: [
{
sourceName: '',
externalFileUrl: '',
externalFileId: ''
}
]
}
});
const {
fields: list,
append,
remove,
update
} = useFieldArray({
control,
name: 'list'
});
useEffect(() => {
if (sources.length > 0) {
reset({
list: sources.map((item) => ({
sourceName: item.sourceName,
externalFileUrl: item.externalFileUrl || '',
externalFileId: item.externalFileId || ''
}))
});
}
}, []);
return (
<Box>
<TableContainer>
<Table bg={'white'}>
<Thead>
<Tr bg={'myGray.50'}>
<Th>{t('dataset:external_url')}</Th>
<Th>{t('dataset:external_id')}</Th>
<Th>{t('dataset:filename')}</Th>
<Th></Th>
</Tr>
</Thead>
<Tbody>
{list.map((item, index) => (
<Tr key={item.id}>
<Td>
<Input
{...register(`list.${index}.externalFileUrl`, {
required: index !== list.length - 1,
onBlur(e) {
const val = (e.target.value || '') as string;
if (val.includes('.') && !list[index]?.sourceName) {
const sourceName = val.split('/').pop() || '';
update(index, {
...list[index],
externalFileUrl: val,
sourceName: decodeURIComponent(sourceName)
});
}
if (val && index === list.length - 1) {
append({
sourceName: '',
externalFileUrl: '',
externalFileId: ''
});
}
}
})}
/>
</Td>
<Td>
<Input {...register(`list.${index}.externalFileId`)} />
</Td>
<Td>
<Input {...register(`list.${index}.sourceName`)} />
</Td>
<Td>
<MyIcon
name={'delete'}
w={'16px'}
cursor={'pointer'}
_hover={{ color: 'red.600' }}
onClick={() => remove(index)}
/>
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
<Flex mt={5} justifyContent={'space-between'}>
<Button
variant={'whitePrimary'}
leftIcon={<SmallAddIcon />}
onClick={() => {
append({
sourceName: '',
externalFileUrl: '',
externalFileId: ''
});
}}
>
{t('common:add_new')}
</Button>
<Button
isDisabled={list.filter((item) => !!item.externalFileUrl).length === 0}
onClick={handleSubmit((data) => {
setSources(
data.list
.filter((item) => !!item.externalFileUrl)
.map((item) => ({
id: getNanoid(32),
createStatus: 'waiting',
sourceName: item.sourceName || item.externalFileUrl,
icon: getFileIcon(item.externalFileUrl),
externalFileId: item.externalFileId,
externalFileUrl: item.externalFileUrl
}))
);
goToNext();
})}
>
{t('common:common.Next Step')}
</Button>
</Flex>
</Box>
);
};

106
projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileCustomText.tsx Normal file
View File

@@ -0,0 +1,106 @@
import React, { useCallback, useEffect } from 'react';
import dynamic from 'next/dynamic';
import { useTranslation } from 'next-i18next';
import { useForm } from 'react-hook-form';
import { Box, Button, Flex, Input, Textarea } from '@chakra-ui/react';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const CustomTet = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <CustomTextInput />}
{activeStep === 1 && <DataProcess showPreviewChunks />}
{activeStep === 2 && <Upload />}
</>
);
};
export default React.memo(CustomTet);
const CustomTextInput = () => {
const { t } = useTranslation();
const { sources, goToNext, setSources } = useContextSelector(DatasetImportContext, (v) => v);
const { register, reset, handleSubmit } = useForm({
defaultValues: {
name: '',
value: ''
}
});
const onSubmit = useCallback(
(data: { name: string; value: string }) => {
const fileId = getNanoid(32);
setSources([
{
id: fileId,
createStatus: 'waiting',
rawText: data.value,
sourceName: data.name,
icon: 'file/fill/manual'
}
]);
goToNext();
},
[goToNext, setSources]
);
useEffect(() => {
const source = sources[0];
if (source) {
reset({
name: source.sourceName,
value: source.rawText
});
}
}, []);
return (
<Box maxW={['100%', '800px']}>
<Box display={['block', 'flex']} alignItems={'center'}>
<Box flex={'0 0 120px'} fontSize={'sm'}>
{t('common:core.dataset.collection.Collection name')}
</Box>
<Input
flex={'1 0 0'}
maxW={['100%', '350px']}
{...register('name', {
required: true
})}
placeholder={t('common:core.dataset.collection.Collection name')}
bg={'myGray.50'}
/>
</Box>
<Box display={['block', 'flex']} alignItems={'flex-start'} mt={5}>
<Box flex={'0 0 120px'} fontSize={'sm'}>
{t('common:core.dataset.collection.Collection raw text')}
</Box>
<Textarea
flex={'1 0 0'}
w={'100%'}
rows={15}
placeholder={t('common:core.dataset.collection.Collection raw text')}
{...register('value', {
required: true
})}
bg={'myGray.50'}
/>
</Box>
<Flex mt={5} justifyContent={'flex-end'}>
<Button onClick={handleSubmit((data) => onSubmit(data))}>
{t('common:common.Next Step')}
</Button>
</Flex>
</Box>
);
};

153
projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLink.tsx Normal file
View File

@@ -0,0 +1,153 @@
import React, { useEffect } from 'react';
import dynamic from 'next/dynamic';
import { useTranslation } from 'next-i18next';
import { useForm } from 'react-hook-form';
import { Box, Button, Flex, Input, Link, Textarea } from '@chakra-ui/react';
import { getNanoid } from '@fastgpt/global/common/string/tools';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { LinkCollectionIcon } from '@fastgpt/global/core/dataset/constants';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { getDocPath } from '@/web/common/system/doc';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const LinkCollection = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <CustomLinkImport />}
{activeStep === 1 && <DataProcess showPreviewChunks />}
{activeStep === 2 && <Upload />}
</>
);
};
export default React.memo(LinkCollection);
const CustomLinkImport = () => {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const { goToNext, sources, setSources, processParamsForm } = useContextSelector(
DatasetImportContext,
(v) => v
);
const { register, reset, handleSubmit, watch } = useForm({
defaultValues: {
link: ''
}
});
const link = watch('link');
const linkList = link.split('\n').filter((item) => item);
useEffect(() => {
reset({
link: sources
.map((item) => item.link)
.filter((item) => item)
.join('\n')
});
}, []);
return (
<Box maxW={['100%', '800px']}>
<Box display={['block', 'flex']} alignItems={'flex-start'} mt={1}>
<Box flex={'0 0 100px'} fontSize={'sm'}>
{t('common:core.dataset.import.Link name')}
</Box>
<Textarea
flex={'1 0 0'}
w={'100%'}
rows={10}
placeholder={t('common:core.dataset.import.Link name placeholder')}
bg={'myGray.50'}
overflowX={'auto'}
whiteSpace={'nowrap'}
{...register('link', {
required: true
})}
/>
</Box>
<Box display={['block', 'flex']} alignItems={'center'} mt={4}>
<Box flex={'0 0 100px'} fontSize={'sm'}>
{t('common:core.dataset.website.Selector')}
<Box color={'myGray.500'} fontSize={'sm'}>
{feConfigs?.docUrl && (
<Link
href={getDocPath('/docs/guide/knowledge_base/websync/#选择器如何使用')}
target="_blank"
>
{t('common:core.dataset.website.Selector Course')}
</Link>
)}
</Box>
</Box>
<Input
flex={'1 0 0'}
maxW={['100%', '350px']}
{...processParamsForm.register('webSelector')}
placeholder={'body .content #document'}
bg={'myGray.50'}
/>
</Box>
<Flex my={4} flexWrap={'wrap'} gap={4} alignItems={'center'} pl={[0, '100px']}>
{linkList.map((item, i) => (
<Flex
key={`${item}-${i}`}
alignItems={'center'}
px={4}
py={2}
borderRadius={'md'}
bg={'myGray.100'}
>
<MyIcon name={LinkCollectionIcon} w={'16px'} />
<Box ml={1} mr={3} wordBreak={'break-all'}>
{item}
</Box>
<MyIcon
name={'common/closeLight'}
w={'14px'}
color={'myGray.500'}
cursor={'pointer'}
onClick={() => {
const newLinkList = linkList.filter((link, index) => index !== i);
reset({
link: newLinkList.join('\n')
});
}}
/>
</Flex>
))}
</Flex>
<Flex mt={5} justifyContent={'flex-end'}>
<Button
onClick={handleSubmit((data) => {
const newLinkList = data.link.split('\n').filter((item) => item);
setSources(
newLinkList.map((link) => ({
id: getNanoid(32),
createStatus: 'waiting',
link,
sourceName: link,
icon: LinkCollectionIcon
}))
);
goToNext();
})}
>
{t('common:common.Next Step')}
</Button>
</Flex>
</Box>
);
};

79
projects/app/src/pageComponents/dataset/detail/Import/diffSource/FileLocal.tsx Normal file
View File

@@ -0,0 +1,79 @@
import React, { useCallback, useEffect, useMemo, useState } from 'react';
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
import { Box, Button } from '@chakra-ui/react';
import FileSelector from '../components/FileSelector';
import { useTranslation } from 'next-i18next';
import dynamic from 'next/dynamic';
import Loading from '@fastgpt/web/components/common/MyLoading';
import { RenderUploadFiles } from '../components/RenderFiles';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const fileType = '.txt, .docx, .csv, .xlsx, .pdf, .md, .html, .pptx';
const FileLocal = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <SelectFile />}
{activeStep === 1 && <DataProcess showPreviewChunks />}
{activeStep === 2 && <Upload />}
</>
);
};
export default React.memo(FileLocal);
const SelectFile = React.memo(function SelectFile() {
const { t } = useTranslation();
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>(
sources.map((source) => ({
isUploading: false,
...source
}))
);
const [uploading, setUploading] = useState(false);
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
useEffect(() => {
setSources(successFiles);
}, [setSources, successFiles]);
const onclickNext = useCallback(() => {
// filter uploaded files
setSelectFiles((state) => state.filter((item) => item.dbFileId));
goToNext();
}, [goToNext]);
return (
<Box>
<FileSelector
fileType={fileType}
selectFiles={selectFiles}
setSelectFiles={setSelectFiles}
onStartSelect={() => setUploading(true)}
onFinishSelect={() => setUploading(false)}
/>
{/* render files */}
<RenderUploadFiles files={selectFiles} setFiles={setSelectFiles} showPreviewContent />
<Box textAlign={'right'} mt={5}>
<Button isDisabled={successFiles.length === 0 || uploading} onClick={onclickNext}>
{selectFiles.length > 0
? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
: ''}
{t('common:common.Next Step')}
</Button>
</Box>
</Box>
);
});

65
projects/app/src/pageComponents/dataset/detail/Import/diffSource/ReTraining.tsx Normal file
View File

@@ -0,0 +1,65 @@
import React from 'react';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
import dynamic from 'next/dynamic';
import DataProcess from '../commonProgress/DataProcess';
import { useRouter } from 'next/router';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { getDatasetCollectionById } from '@/web/core/dataset/api';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { ImportProcessWayEnum } from '@/web/core/dataset/constants';
import { getCollectionIcon } from '@fastgpt/global/core/dataset/utils';
const Upload = dynamic(() => import('../commonProgress/Upload'));
const ReTraining = () => {
const router = useRouter();
const { collectionId = '' } = router.query as {
collectionId: string;
};
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
const setSources = useContextSelector(DatasetImportContext, (v) => v.setSources);
const processParamsForm = useContextSelector(DatasetImportContext, (v) => v.processParamsForm);
const { loading } = useRequest2(() => getDatasetCollectionById(collectionId), {
refreshDeps: [collectionId],
manual: false,
onSuccess: (collection) => {
setSources([
{
dbFileId: collection.fileId,
link: collection.rawLink,
apiFileId: collection.apiFileId,
createStatus: 'waiting',
icon: getCollectionIcon(collection.type, collection.name),
id: collection._id,
isUploading: false,
sourceName: collection.name,
uploadedFileRate: 100
}
]);
processParamsForm.reset({
mode: collection.trainingType,
way: ImportProcessWayEnum.auto,
embeddingChunkSize: collection.chunkSize,
qaChunkSize: collection.chunkSize,
customSplitChar: collection.chunkSplitter,
qaPrompt: collection.qaPrompt,
webSelector: collection.metadata?.webSelector
});
}
});
return (
<MyBox isLoading={loading} h={'100%'} overflow={'auto'}>
{activeStep === 0 && <DataProcess showPreviewChunks={true} />}
{activeStep === 1 && <Upload />}
</MyBox>
);
};
export default React.memo(ReTraining);

101
projects/app/src/pageComponents/dataset/detail/Import/diffSource/TableLocal.tsx Normal file
View File

@@ -0,0 +1,101 @@
import React, { useEffect, useMemo, useState } from 'react';
import { ImportSourceItemType } from '@/web/core/dataset/type.d';
import { Box, Button } from '@chakra-ui/react';
import FileSelector from '../components/FileSelector';
import { useTranslation } from 'next-i18next';
import dynamic from 'next/dynamic';
import { fileDownload } from '@/web/common/file/utils';
import { RenderUploadFiles } from '../components/RenderFiles';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
const Upload = dynamic(() => import('../commonProgress/Upload'));
const fileType = '.csv';
const FileLocal = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
return (
<>
{activeStep === 0 && <SelectFile />}
{activeStep === 1 && <PreviewData showPreviewChunks />}
{activeStep === 2 && <Upload />}
</>
);
};
export default React.memo(FileLocal);
const csvTemplate = `index,content
"第一列内容","第二列内容"
"必填列","可选列。CSV 中请注意内容不能包含双引号，双引号是列分割符号"
"只会将第一和第二列内容导入，其余列会被忽略",""
"结合人工智能的演进历程,AIGC的发展大致可以分为三个阶段，即:早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期),以及快速发展展阶段(21世纪10年代中期至今)。",""
"AIGC发展分为几个阶段？","早期萌芽阶段(20世纪50年代至90年代中期)、沉淀积累阶段(20世纪90年代中期至21世纪10年代中期)、快速发展展阶段(21世纪10年代中期至今)"`;
const SelectFile = React.memo(function SelectFile() {
const { t } = useTranslation();
const { goToNext, sources, setSources } = useContextSelector(DatasetImportContext, (v) => v);
const [selectFiles, setSelectFiles] = useState<ImportSourceItemType[]>(
sources.map((source) => ({
isUploading: false,
...source
}))
);
const [uploading, setUploading] = useState(false);
const successFiles = useMemo(() => selectFiles.filter((item) => !item.errorMsg), [selectFiles]);
useEffect(() => {
setSources(successFiles);
}, [successFiles]);
return (
<Box>
<FileSelector
fileType={fileType}
selectFiles={selectFiles}
setSelectFiles={setSelectFiles}
onStartSelect={() => setUploading(true)}
onFinishSelect={() => setUploading(false)}
/>
<Box
mt={4}
color={'primary.600'}
textDecoration={'underline'}
cursor={'pointer'}
onClick={() =>
fileDownload({
text: csvTemplate,
type: 'text/csv;charset=utf-8',
filename: 'template.csv'
})
}
>
{t('common:core.dataset.import.Down load csv template')}
</Box>
{/* render files */}
<RenderUploadFiles files={selectFiles} setFiles={setSelectFiles} />
<Box textAlign={'right'} mt={5}>
<Button
isDisabled={successFiles.length === 0 || uploading}
onClick={() => {
setSelectFiles((state) => state.filter((item) => !item.errorMsg));
goToNext();
}}
>
{selectFiles.length > 0
? `${t('core.dataset.import.Total files', { total: selectFiles.length })} | `
: ''}
{t('common:common.Next Step')}
</Button>
</Box>
</Box>
);
});

53
projects/app/src/pageComponents/dataset/detail/Import/index.tsx Normal file
View File

@@ -0,0 +1,53 @@
import React, { useMemo } from 'react';
import { Box, Flex } from '@chakra-ui/react';
import dynamic from 'next/dynamic';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { useContextSelector } from 'use-context-selector';
import DatasetImportContextProvider, { DatasetImportContext } from './Context';
const FileLocal = dynamic(() => import('./diffSource/FileLocal'));
const FileLink = dynamic(() => import('./diffSource/FileLink'));
const FileCustomText = dynamic(() => import('./diffSource/FileCustomText'));
const TableLocal = dynamic(() => import('./diffSource/TableLocal'));
const ExternalFileCollection = dynamic(() => import('./diffSource/ExternalFile'));
const APIDatasetCollection = dynamic(() => import('./diffSource/APIDataset'));
const ReTraining = dynamic(() => import('./diffSource/ReTraining'));
const ImportDataset = () => {
const importSource = useContextSelector(DatasetImportContext, (v) => v.importSource);
const ImportComponent = useMemo(() => {
if (importSource === ImportDataSourceEnum.reTraining) return ReTraining;
if (importSource === ImportDataSourceEnum.fileLocal) return FileLocal;
if (importSource === ImportDataSourceEnum.fileLink) return FileLink;
if (importSource === ImportDataSourceEnum.fileCustom) return FileCustomText;
if (importSource === ImportDataSourceEnum.csvTable) return TableLocal;
if (importSource === ImportDataSourceEnum.externalFile) return ExternalFileCollection;
if (importSource === ImportDataSourceEnum.apiDataset) return APIDatasetCollection;
}, [importSource]);
return ImportComponent ? (
<Box flex={'1 0 0'} overflow={'auto'}>
<ImportComponent />
</Box>
) : null;
};
const Render = () => {
return (
<Flex
flexDirection={'column'}
bg={'white'}
h={'100%'}
px={[2, 9]}
py={[2, 5]}
borderRadius={'md'}
>
<DatasetImportContextProvider>
<ImportDataset />
</DatasetImportContextProvider>
</Flex>
);
};
export default React.memo(Render);

7
projects/app/src/pageComponents/dataset/detail/Import/type.d.ts vendored Normal file
View File

@@ -0,0 +1,7 @@
import { ImportSourceItemType } from '@/web/core/dataset/type';
export type UploadFileItemType = ImportSourceItemType & {
file?: File;
isUploading: boolean;
uploadedFileRate: number;
};

23
projects/app/src/pageComponents/dataset/detail/Import/utils.tsx Normal file
View File

@@ -0,0 +1,23 @@
import { ImportSourceItemType } from '@/web/core/dataset/type';
import { DatasetSourceReadTypeEnum } from '@fastgpt/global/core/dataset/constants';
export const getPreviewSourceReadType = (previewSource: ImportSourceItemType) => {
if (previewSource.dbFileId) {
return DatasetSourceReadTypeEnum.fileLocal;
}
if (previewSource.link) {
return DatasetSourceReadTypeEnum.link;
}
if (previewSource.apiFileId) {
return DatasetSourceReadTypeEnum.apiFile;
}
if (previewSource.externalFileId) {
return DatasetSourceReadTypeEnum.externalFile;
}
return DatasetSourceReadTypeEnum.fileLocal;
};
export default function Dom() {
return <></>;
}

84
projects/app/src/pageComponents/dataset/detail/Info/components/EditApiServiceModal.tsx Normal file
View File

@@ -0,0 +1,84 @@
import React from 'react';
import { ModalFooter, ModalBody, Button, Flex } from '@chakra-ui/react';
import MyModal from '@fastgpt/web/components/common/MyModal/index';
import { useTranslation } from 'next-i18next';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useForm } from 'react-hook-form';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { APIFileServer, FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
import ApiDatasetForm from '@/pageComponents/dataset/ApiDatasetForm';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { datasetTypeCourseMap } from '@/web/core/dataset/constants';
import { getDocPath } from '@/web/common/system/doc';
import MyIcon from '@fastgpt/web/components/common/Icon';
export type EditAPIDatasetInfoFormType = {
id: string;
apiServer?: APIFileServer;
yuqueServer?: YuqueServer;
feishuServer?: FeishuServer;
};
const EditAPIDatasetInfoModal = ({
onClose,
onEdit,
title,
...defaultForm
}: EditAPIDatasetInfoFormType & {
title: string;
onClose: () => void;
onEdit: (data: EditAPIDatasetInfoFormType) => any;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const type = datasetDetail.type;
const form = useForm<EditAPIDatasetInfoFormType>({
defaultValues: defaultForm
});
const { runAsync: onSave, loading } = useRequest2(
(data: EditAPIDatasetInfoFormType) => onEdit(data),
{
onSuccess: (res) => {
toast({
title: t('common:common.Update Success'),
status: 'success'
});
onClose();
}
}
);
return (
<MyModal isOpen onClose={onClose} w={'450px'} iconSrc="modal/edit" title={title}>
<ModalBody>
{datasetTypeCourseMap[type] && (
<Flex
alignItems={'center'}
justifyContent={'flex-end'}
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() => window.open(getDocPath(datasetTypeCourseMap[type]), '_blank')}
>
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}
</Flex>
)}
{/* @ts-ignore */}
<ApiDatasetForm type={type} form={form} />
</ModalBody>
<ModalFooter>
<Button isLoading={loading} onClick={form.handleSubmit(onSave)} px={6}>
{t('common:common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};
export default EditAPIDatasetInfoModal;

426
projects/app/src/pageComponents/dataset/detail/Info/index.tsx Normal file
View File

@@ -0,0 +1,426 @@
import React, { useEffect, useState } from 'react';
import { Box, Flex, Switch, Input } from '@chakra-ui/react';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { useForm } from 'react-hook-form';
import type { DatasetItemType } from '@fastgpt/global/core/dataset/type.d';
import Avatar from '@fastgpt/web/components/common/Avatar';
import { useTranslation } from 'next-i18next';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import AIModelSelector from '@/components/Select/AIModelSelector';
import { postRebuildEmbedding } from '@/web/core/dataset/api';
import type { EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.d';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import MyDivider from '@fastgpt/web/components/common/MyDivider/index';
import { DatasetTypeEnum, DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { DatasetPermissionList } from '@fastgpt/global/support/permission/dataset/constant';
import MemberManager from '../../MemberManager';
import {
getCollaboratorList,
postUpdateDatasetCollaborators,
deleteDatasetCollaborators
} from '@/web/core/dataset/api/collaborator';
import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag';
import dynamic from 'next/dynamic';
import type { EditAPIDatasetInfoFormType } from './components/EditApiServiceModal';
import { EditResourceInfoFormType } from '@/components/common/Modal/EditResourceModal';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
const EditResourceModal = dynamic(() => import('@/components/common/Modal/EditResourceModal'));
const EditAPIDatasetInfoModal = dynamic(() => import('./components/EditApiServiceModal'));
const Info = ({ datasetId }: { datasetId: string }) => {
const { t } = useTranslation();
const { datasetDetail, loadDatasetDetail, updateDataset, rebuildingCount, trainingCount } =
useContextSelector(DatasetPageContext, (v) => v);
const [editedDataset, setEditedDataset] = useState<EditResourceInfoFormType>();
const [editedAPIDataset, setEditedAPIDataset] = useState<EditAPIDatasetInfoFormType>();
const refetchDatasetTraining = useContextSelector(
DatasetPageContext,
(v) => v.refetchDatasetTraining
);
const { setValue, register, handleSubmit, watch, reset } = useForm<DatasetItemType>({
defaultValues: datasetDetail
});
const vectorModel = watch('vectorModel');
const agentModel = watch('agentModel');
const { feConfigs, datasetModelList, embeddingModelList } = useSystemStore();
const { ConfirmModal: ConfirmDelModal } = useConfirm({
content: t('common:core.dataset.Delete Confirm'),
type: 'delete'
});
const { openConfirm: onOpenConfirmRebuild, ConfirmModal: ConfirmRebuildModal } = useConfirm({
title: t('common:common.confirm.Common Tip'),
content: t('dataset:confirm_to_rebuild_embedding_tip'),
type: 'delete'
});
const { openConfirm: onOpenConfirmSyncSchedule, ConfirmModal: ConfirmSyncScheduleModal } =
useConfirm({
title: t('common:common.confirm.Common Tip')
});
const { runAsync: onSave } = useRequest2(
(data: DatasetItemType) => {
return updateDataset({
id: datasetId,
agentModel: data.agentModel,
externalReadUrl: data.externalReadUrl
});
},
{
successToast: t('common:common.Update Success'),
errorToast: t('common:common.Update Failed')
}
);
const { runAsync: onRebuilding } = useRequest2(
(vectorModel: EmbeddingModelItemType) => {
return postRebuildEmbedding({
datasetId,
vectorModel: vectorModel.model
});
},
{
onSuccess() {
refetchDatasetTraining();
loadDatasetDetail(datasetId);
},
successToast: t('dataset:rebuild_embedding_start_tip'),
errorToast: t('common:common.Update Failed')
}
);
const { runAsync: onEditBaseInfo } = useRequest2(updateDataset, {
onSuccess() {
setEditedDataset(undefined);
},
successToast: t('common:common.Update Success'),
errorToast: t('common:common.Update Failed')
});
useEffect(() => {
reset(datasetDetail);
}, [datasetDetail, datasetDetail._id, reset]);
const isTraining = rebuildingCount > 0 || trainingCount > 0;
return (
<Box w={'100%'} h={'100%'} p={6}>
<Box>
<Flex mb={2} alignItems={'center'}>
<Avatar src={datasetDetail.avatar} w={'20px'} h={'20px'} borderRadius={'xs'} />
<Box ml={1.5}>
<Box fontWeight={'bold'} color={'myGray.900'}>
{datasetDetail.name}
</Box>
</Box>
<MyIcon
pl={1.5}
name={'edit'}
_hover={{ color: 'primary.600' }}
w={'0.875rem'}
cursor={'pointer'}
onClick={() =>
setEditedDataset({
id: datasetDetail._id,
name: datasetDetail.name,
avatar: datasetDetail.avatar,
intro: datasetDetail.intro
})
}
/>
</Flex>
{DatasetTypeMap[datasetDetail.type] && (
<Flex alignItems={'center'} justifyContent={'space-between'}>
<DatasetTypeTag type={datasetDetail.type} />
</Flex>
)}
<Box
flex={1}
className={'textEllipsis3'}
pt={3}
wordBreak={'break-all'}
fontSize={'xs'}
color={'myGray.500'}
>
{datasetDetail.intro || t('common:core.dataset.Intro Placeholder')}
</Box>
</Box>
<MyDivider my={4} h={'2px'} maxW={'500px'} />
<Box>
<Flex w={'100%'} flexDir={'column'}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('common:core.dataset.Dataset ID')}
</FormLabel>
<Box fontSize={'mini'}>{datasetDetail._id}</Box>
</Flex>
<Box mt={5} w={'100%'}>
<Flex alignItems={'center'} fontSize={'mini'}>
<FormLabel fontWeight={'500'} flex={'1 0 0'}>
{t('common:core.ai.model.Vector Model')}
</FormLabel>
<MyTooltip label={t('dataset:vector_model_max_tokens_tip')}>
<Box>
{t('dataset:chunk_max_tokens')}: {vectorModel.maxToken}
</Box>
</MyTooltip>
</Flex>
<Box pt={2}>
<AIModelSelector
w={'100%'}
value={vectorModel.model}
fontSize={'mini'}
disableTip={
isTraining
? t(
'dataset:the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt'
)
: undefined
}
list={embeddingModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
const vectorModel = embeddingModelList.find((item) => item.model === e);
if (!vectorModel) return;
return onOpenConfirmRebuild(async () => {
await onRebuilding(vectorModel);
setValue('vectorModel', vectorModel);
})();
}}
/>
</Box>
</Box>
<Box pt={5}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('common:core.ai.model.Dataset Agent Model')}
</FormLabel>
<Box pt={2}>
<AIModelSelector
w={'100%'}
value={agentModel.model}
list={datasetModelList.map((item) => ({
label: item.name,
value: item.model
}))}
fontSize={'mini'}
onchange={(e) => {
const agentModel = datasetModelList.find((item) => item.model === e);
if (!agentModel) return;
setValue('agentModel', agentModel);
return handleSubmit((data) => onSave({ ...data, agentModel: agentModel }))();
}}
/>
</Box>
</Box>
{feConfigs?.isPlus && (
<Flex alignItems={'center'} pt={5}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:sync_schedule')}
</FormLabel>
<QuestionTip ml={1} label={t('dataset:sync_schedule_tip')} />
<Box flex={1} />
<Switch
isChecked={!!datasetDetail.autoSync}
onChange={(e) => {
e.preventDefault();
const autoSync = e.target.checked;
const text = autoSync ? t('dataset:open_auto_sync') : t('dataset:close_auto_sync');
onOpenConfirmSyncSchedule(
async () => {
return updateDataset({
id: datasetId,
autoSync
});
},
undefined,
text
)();
}}
/>
</Flex>
)}
{datasetDetail.type === DatasetTypeEnum.externalFile && (
<>
<Box w={'100%'} alignItems={'center'} pt={4}>
<FormLabel display={'flex'} pb={2} fontSize={'mini'} fontWeight={'500'}>
<Box>{t('dataset:external_read_url')}</Box>
<QuestionTip label={t('dataset:external_read_url_tip')} />
</FormLabel>
<Input
fontSize={'mini'}
flex={[1, '0 0 320px']}
placeholder="https://test.com/read?fileId={{fileId}}"
{...register('externalReadUrl')}
onBlur={handleSubmit((data) => onSave(data))}
/>
</Box>
</>
)}
{datasetDetail.type === DatasetTypeEnum.apiDataset && (
<>
<Box w={'100%'} alignItems={'center'} pt={4}>
<Flex justifyContent={'space-between'} mb={1}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:api_url')}
</FormLabel>
<MyIcon
name={'edit'}
w={'14px'}
_hover={{ color: 'primary.600' }}
cursor={'pointer'}
onClick={() =>
setEditedAPIDataset({
id: datasetDetail._id,
apiServer: datasetDetail.apiServer
})
}
/>
</Flex>
<Box fontSize={'mini'}>{datasetDetail.apiServer?.baseUrl}</Box>
</Box>
</>
)}
{datasetDetail.type === DatasetTypeEnum.yuque && (
<>
<Box w={'100%'} alignItems={'center'} pt={4}>
<Flex justifyContent={'space-between'} mb={1}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:yuque_dataset_config')}
</FormLabel>
<MyIcon
name={'edit'}
w={'14px'}
_hover={{ color: 'primary.600' }}
cursor={'pointer'}
onClick={() =>
setEditedAPIDataset({
id: datasetDetail._id,
yuqueServer: datasetDetail.yuqueServer
})
}
/>
</Flex>
<Box fontSize={'mini'}>{datasetDetail.yuqueServer?.userId}</Box>
</Box>
</>
)}
{datasetDetail.type === DatasetTypeEnum.feishu && (
<>
<Box w={'100%'} alignItems={'center'} pt={4}>
<Flex justifyContent={'space-between'} mb={1}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:feishu_dataset_config')}
</FormLabel>
<MyIcon
name={'edit'}
w={'14px'}
_hover={{ color: 'primary.600' }}
cursor={'pointer'}
onClick={() =>
setEditedAPIDataset({
id: datasetDetail._id,
feishuServer: datasetDetail.feishuServer
})
}
/>
</Flex>
<Box fontSize={'mini'}>{datasetDetail.feishuServer?.folderToken}</Box>
</Box>
</>
)}
</Box>
{datasetDetail.permission.hasManagePer && (
<>
<MyDivider my={4} h={'2px'} maxW={'500px'} />
<Box>
<MemberManager
managePer={{
permission: datasetDetail.permission,
onGetCollaboratorList: () => getCollaboratorList(datasetId),
permissionList: DatasetPermissionList,
onUpdateCollaborators: (body) =>
postUpdateDatasetCollaborators({
...body,
datasetId
}),
onDelOneCollaborator: async ({ groupId, tmbId, orgId }) => {
if (tmbId) {
return deleteDatasetCollaborators({
datasetId,
tmbId
});
} else if (groupId) {
return deleteDatasetCollaborators({
datasetId,
groupId
});
} else if (orgId) {
return deleteDatasetCollaborators({
datasetId,
orgId
});
}
}
}}
/>
</Box>
</>
)}
<ConfirmDelModal />
<ConfirmRebuildModal countDown={10} />
<ConfirmSyncScheduleModal />
{editedDataset && (
<EditResourceModal
{...editedDataset}
title={t('common:dataset.Edit Info')}
onClose={() => setEditedDataset(undefined)}
onEdit={(data) =>
onEditBaseInfo({
id: editedDataset.id,
name: data.name,
intro: data.intro,
avatar: data.avatar
})
}
/>
)}
{editedAPIDataset && (
<EditAPIDatasetInfoModal
{...editedAPIDataset}
title={t('dataset:edit_dataset_config')}
onClose={() => setEditedAPIDataset(undefined)}
onEdit={(data) =>
updateDataset({
id: datasetId,
apiServer: data.apiServer,
yuqueServer: data.yuqueServer,
feishuServer: data.feishuServer
})
}
/>
)}
</Box>
);
};
export default React.memo(Info);

558
projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx Normal file
View File

@@ -0,0 +1,558 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Box, Flex, Button, Textarea, useTheme, Grid, HStack } from '@chakra-ui/react';
import {
Control,
FieldArrayWithId,
UseFieldArrayAppend,
UseFieldArrayRemove,
UseFormRegister,
useFieldArray,
useForm
} from 'react-hook-form';
import {
postInsertData2Dataset,
putDatasetDataById,
delOneDatasetDataById,
getDatasetCollectionById,
getDatasetDataItemById
} from '@/web/core/dataset/api';
import { useToast } from '@fastgpt/web/hooks/useToast';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useQuery } from '@tanstack/react-query';
import { useTranslation } from 'next-i18next';
import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { getDefaultIndex, getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
import DeleteIcon from '@fastgpt/web/components/common/Icon/delete';
import { defaultCollectionDetail } from '@/web/core/dataset/constants';
import { getDocPath } from '@/web/common/system/doc';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import LightRowTabs from '@fastgpt/web/components/common/Tabs/LightRowTabs';
import styles from './styles.module.scss';
export type InputDataType = {
q: string;
a: string;
indexes: (Omit<DatasetDataIndexItemType, 'dataId'> & {
dataId?: string; // pg data id
})[];
};
enum TabEnum {
content = 'content',
index = 'index'
}
const InputDataModal = ({
collectionId,
dataId,
defaultValue,
onClose,
onSuccess
}: {
collectionId: string;
dataId?: string;
defaultValue?: { q: string; a?: string };
onClose: () => void;
onSuccess: (data: InputDataType & { dataId: string }) => void;
}) => {
const { t } = useTranslation();
const theme = useTheme();
const { toast } = useToast();
const [currentTab, setCurrentTab] = useState(TabEnum.content);
const { embeddingModelList, defaultModels } = useSystemStore();
const { isPc } = useSystem();
const { register, handleSubmit, reset, control } = useForm<InputDataType>();
const {
fields: indexes,
append: appendIndexes,
remove: removeIndexes
} = useFieldArray({
control,
name: 'indexes'
});
const tabList = [
{
label: (
<Flex align={'center'}>
<Box>{t('common:dataset.data.edit.divide_content')}</Box>
</Flex>
),
value: TabEnum.content
},
{
label: (
<Flex align={'center'}>
<Box>{t('common:dataset.data.edit.Index', { amount: indexes.length })}</Box>
<MyTooltip label={t('common:core.app.tool_label.view_doc')}>
<MyIcon
name={'book'}
w={'1rem'}
mr={'0.38rem'}
color={'myGray.500'}
ml={1}
onClick={() =>
window.open(getDocPath('/docs/guide/knowledge_base/dataset_engine/'), '_blank')
}
_hover={{
color: 'primary.600',
cursor: 'pointer'
}}
/>
</MyTooltip>
</Flex>
),
value: TabEnum.index
}
];
const { ConfirmModal, openConfirm } = useConfirm({
content: t('common:dataset.data.Delete Tip'),
type: 'delete'
});
const { data: collection = defaultCollectionDetail } = useQuery(
['loadCollectionId', collectionId],
() => {
return getDatasetCollectionById(collectionId);
}
);
const { isFetching: isFetchingData } = useQuery(
['getDatasetDataItemById', dataId],
() => {
if (dataId) return getDatasetDataItemById(dataId);
return null;
},
{
onSuccess(res) {
if (res) {
reset({
q: res.q,
a: res.a,
indexes: res.indexes
});
} else if (defaultValue) {
reset({
q: defaultValue.q,
a: defaultValue.a
});
}
},
onError(err) {
toast({
status: 'error',
title: t(getErrText(err) as any)
});
onClose();
}
}
);
const maxToken = useMemo(() => {
const vectorModel =
embeddingModelList.find((item) => item.model === collection.dataset.vectorModel) ||
defaultModels.embedding;
return vectorModel?.maxToken || 3000;
}, [collection.dataset.vectorModel, defaultModels.embedding, embeddingModelList]);
// import new data
const { mutate: sureImportData, isLoading: isImporting } = useRequest({
mutationFn: async (e: InputDataType) => {
if (!e.q) {
setCurrentTab(TabEnum.content);
return Promise.reject(t('common:dataset.data.input is empty'));
}
const totalLength = e.q.length + (e.a?.length || 0);
if (totalLength >= maxToken * 1.4) {
return Promise.reject(t('common:core.dataset.data.Too Long'));
}
const data = { ...e };
const dataId = await postInsertData2Dataset({
collectionId: collection._id,
q: e.q,
a: e.a,
// remove dataId
indexes:
e.indexes?.map((index) => ({
...index,
dataId: undefined
})) || []
});
return {
...data,
dataId
};
},
successToast: t('common:dataset.data.Input Success Tip'),
onSuccess(e) {
reset({
...e,
q: '',
a: '',
indexes: []
});
onSuccess(e);
},
errorToast: t('common:common.error.unKnow')
});
// update
const { runAsync: onUpdateData, loading: isUpdating } = useRequest2(
async (e: InputDataType) => {
if (!dataId) return Promise.reject(t('common:common.error.unKnow'));
// not exactly same
await putDatasetDataById({
dataId,
...e,
indexes:
e.indexes?.map((index) =>
index.defaultIndex ? getDefaultIndex({ q: e.q, a: e.a, dataId: index.dataId }) : index
) || []
});
return {
dataId,
...e
};
},
{
successToast: t('common:dataset.data.Update Success Tip'),
onSuccess(data) {
onSuccess(data);
onClose();
}
}
);
const isLoading = isFetchingData;
const icon = useMemo(
() => getSourceNameIcon({ sourceName: collection.sourceName, sourceId: collection.sourceId }),
[collection]
);
return (
<MyModal
isOpen={true}
isCentered
w={['20rem', '64rem']}
onClose={() => onClose()}
closeOnOverlayClick={false}
maxW={'1440px'}
h={'46.25rem'}
title={
<Flex ml={-3}>
<MyIcon name={icon as any} w={['16px', '20px']} mr={2} />
<Box
className={'textEllipsis'}
wordBreak={'break-all'}
fontSize={'md'}
maxW={['200px', '50vw']}
fontWeight={'500'}
color={'myGray.900'}
whiteSpace={'nowrap'}
overflow={'hidden'}
textOverflow={'ellipsis'}
>
{collection.sourceName || t('common:common.UnKnow Source')}
</Box>
</Flex>
}
>
<MyBox
display={'flex'}
flexDir={'column'}
isLoading={isLoading}
h={'100%'}
py={[6, '1.5rem']}
px={[5, '3.25rem']}
>
<Flex justify={'space-between'} gap={4} w={'100%'}>
<Flex justify={'space-between'} pb={4}>
<LightRowTabs<TabEnum>
list={tabList}
p={0}
value={currentTab}
onChange={(e: TabEnum) => setCurrentTab(e)}
/>
</Flex>
{currentTab === TabEnum.index && (
<Button
variant={'whiteBase'}
boxShadow={'1'}
p={0}
onClick={() =>
appendIndexes({
defaultIndex: false,
text: '',
dataId: `${Date.now()}`
})
}
>
<Flex px={'0.62rem'} py={2}>
<MyIcon name={'common/addLight'} w={'1rem'} mr={'0.38rem'} />
{t('common:add_new')}
</Flex>
</Button>
)}
</Flex>
<Box w={'100%'} flexGrow={1} overflow={'scroll'}>
{currentTab === TabEnum.content && <InputTab maxToken={maxToken} register={register} />}
{currentTab === TabEnum.index && (
<DataIndex
register={register}
maxToken={maxToken}
appendIndexes={appendIndexes}
removeIndexes={removeIndexes}
indexes={indexes}
/>
)}
</Box>
<Flex justifyContent={'flex-end'} pt={8} pb={[8, 0]} h={[24, 16]}>
<MyTooltip
label={collection.permission.hasWritePer ? '' : t('common:dataset.data.Can not edit')}
>
<Button
isDisabled={!collection.permission.hasWritePer}
isLoading={isImporting || isUpdating}
// @ts-ignore
onClick={handleSubmit(dataId ? onUpdateData : sureImportData)}
>
{dataId ? t('common:common.Confirm Update') : t('common:common.Confirm Import')}
</Button>
</MyTooltip>
</Flex>
</MyBox>
<ConfirmModal />
</MyModal>
);
};
export default React.memo(InputDataModal);
const InputTab = ({
maxToken,
register
}: {
maxToken: number;
register: UseFormRegister<InputDataType>;
}) => {
const { t } = useTranslation();
return (
<>
<Flex h={'100%'} gap={6} flexDir={['column', 'row']} w={'100%'}>
<Flex flexDir={'column'} flex={1}>
<Flex mb={2} fontWeight={'medium'} fontSize={'sm'} alignItems={'center'} h={8}>
<Box color={'red.600'}>*</Box>
<Box color={'myGray.900'}>{t('common:core.dataset.data.Main Content')}</Box>
<QuestionTip label={t('common:core.dataset.data.Data Content Tip')} ml={1} />
</Flex>
<Box
borderRadius={'md'}
border={'1.5px solid var(--Gray-Modern-200, #E8EBF0)'}
bg={'myGray.25'}
flex={1}
>
<Textarea
resize={'none'}
placeholder={t('core.dataset.data.Data Content Placeholder', { maxToken })}
className={styles.scrollbar}
maxLength={maxToken}
h={'100%'}
tabIndex={1}
_focus={{
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
borderColor={'transparent'}
bg={'myGray.25'}
{...register(`q`, {
required: true
})}
/>
</Box>
</Flex>
<Flex flex={1} flexDir={'column'}>
<Flex mb={2} fontWeight={'medium'} fontSize={'sm'} alignItems={'center'} h={8}>
<Box color={'myGray.900'}>{t('common:core.dataset.data.Auxiliary Data')}</Box>
<QuestionTip label={t('common:core.dataset.data.Auxiliary Data Tip')} ml={1} />
</Flex>
<Box
borderRadius={'md'}
border={'1.5px solid '}
borderColor={'myGray.200'}
bg={'myGray.25'}
flex={1}
>
<Textarea
resize={'none'}
placeholder={t('core.dataset.data.Auxiliary Data Placeholder', {
maxToken: maxToken * 1.5
})}
className={styles.scrollbar}
borderColor={'transparent'}
h={'100%'}
tabIndex={1}
bg={'myGray.25'}
maxLength={maxToken * 1.5}
{...register('a')}
/>
</Box>
</Flex>
</Flex>
</>
);
};
const DataIndex = ({
maxToken,
register,
indexes,
appendIndexes,
removeIndexes
}: {
maxToken: number;
register: UseFormRegister<InputDataType>;
indexes: FieldArrayWithId<InputDataType, 'indexes', 'id'>[];
appendIndexes: UseFieldArrayAppend<InputDataType, 'indexes'>;
removeIndexes: UseFieldArrayRemove;
}) => {
const { t } = useTranslation();
return (
<>
<Flex mt={3} gap={3} flexDir={'column'}>
<Box
p={4}
borderRadius={'md'}
border={'1.5px solid var(--light-fastgpt-primary-opacity-01, rgba(51, 112, 255, 0.10))'}
bg={'primary.50'}
>
<Flex mb={2}>
<Box flex={1} fontWeight={'medium'} fontSize={'sm'} color={'primary.700'}>
{t('common:dataset.data.Default Index')}
</Box>
</Flex>
<Box fontSize={'sm'} fontWeight={'medium'} color={'myGray.600'}>
{t('common:core.dataset.data.Default Index Tip')}
</Box>
</Box>
{indexes?.map((index, i) => {
return (
!index.defaultIndex && (
<Box
key={index.dataId || i}
p={4}
borderRadius={'md'}
border={'1.5px solid var(--Gray-Modern-200, #E8EBF0)'}
bg={'myGray.25'}
_hover={{
'& .delete': {
display: 'block'
}
}}
>
<Flex mb={2}>
<Box flex={1} fontWeight={'medium'} fontSize={'sm'} color={'myGray.900'}>
{t('dataset.data.Custom Index Number', { number: i })}
</Box>
<DeleteIcon
onClick={() => {
if (indexes.length <= 1) {
appendIndexes(getDefaultIndex({ dataId: `${Date.now()}` }));
}
removeIndexes(i);
}}
/>
</Flex>
<DataIndexTextArea index={i} maxToken={maxToken} register={register} />
</Box>
)
);
})}
</Flex>
</>
);
};
const DataIndexTextArea = ({
index,
maxToken,
register
}: {
index: number;
maxToken: number;
register: UseFormRegister<InputDataType>;
}) => {
const { t } = useTranslation();
const TextareaDom = useRef<HTMLTextAreaElement | null>(null);
const {
ref: TextareaRef,
required,
name,
onChange: onTextChange,
onBlur
} = register(`indexes.${index}.text`, { required: true });
const textareaMinH = '40px';
useEffect(() => {
if (TextareaDom.current) {
TextareaDom.current.style.height = textareaMinH;
TextareaDom.current.style.height = `${TextareaDom.current.scrollHeight + 5}px`;
}
}, []);
const autoHeight = useCallback((e: React.ChangeEvent<HTMLTextAreaElement>) => {
if (e.target) {
e.target.style.height = textareaMinH;
e.target.style.height = `${e.target.scrollHeight + 5}px`;
}
}, []);
return (
<Textarea
maxLength={maxToken}
borderColor={'transparent'}
className={styles.scrollbar}
minH={textareaMinH}
px={0}
pt={0}
isRequired={required}
whiteSpace={'pre-wrap'}
resize={'none'}
_focus={{
px: 3,
py: 1,
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
placeholder={t('common:dataset.data.Index Placeholder')}
ref={(e) => {
if (e) TextareaDom.current = e;
TextareaRef(e);
}}
required
name={name}
onChange={(e) => {
autoHeight(e);
onTextChange(e);
}}
onFocus={autoHeight}
onBlur={onBlur}
/>
);
};

132
projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx Normal file
View File

@@ -0,0 +1,132 @@
import React, { useMemo } from 'react';
import { Box, Flex, Button } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { getDatasetCollectionById } from '@/web/core/dataset/api';
import { useRouter } from 'next/router';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { formatFileSize } from '@fastgpt/global/common/file/tools';
import { formatTime2YMDHM } from '@fastgpt/global/common/string/time';
import { DatasetCollectionTypeMap, TrainingTypeMap } from '@fastgpt/global/core/dataset/constants';
import { getCollectionSourceAndOpen } from '@/web/core/dataset/hooks/readCollectionSource';
import MyIcon from '@fastgpt/web/components/common/Icon';
const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
const { t } = useTranslation();
const router = useRouter();
const { collectionId = '' } = router.query as {
collectionId: string;
datasetId: string;
};
const readSource = getCollectionSourceAndOpen({
collectionId
});
const { data: collection, loading: isLoading } = useRequest2(
() => getDatasetCollectionById(collectionId),
{
onError: () => {
router.replace({
query: {
datasetId
}
});
},
manual: false
}
);
const metadataList = useMemo<{ label?: string; value?: any }[]>(() => {
if (!collection) return [];
const webSelector = collection?.metadata?.webPageSelector;
return [
{
label: t('common:core.dataset.collection.metadata.source'),
value: t(DatasetCollectionTypeMap[collection.type]?.name as any)
},
{
label: t('common:core.dataset.collection.metadata.source name'),
value: collection.file?.filename || collection?.rawLink || collection?.name
},
{
label: t('common:core.dataset.collection.metadata.source size'),
value: collection.file ? formatFileSize(collection.file.length) : '-'
},
{
label: t('common:core.dataset.collection.metadata.Createtime'),
value: formatTime2YMDHM(collection.createTime)
},
{
label: t('common:core.dataset.collection.metadata.Updatetime'),
value: formatTime2YMDHM(collection.updateTime)
},
{
label: t('common:core.dataset.collection.metadata.Raw text length'),
value: collection.rawTextLength ?? '-'
},
{
label: t('dataset:collection.Training type'),
value: t(TrainingTypeMap[collection.trainingType]?.label as any)
},
{
label: t('common:core.dataset.collection.metadata.Chunk Size'),
value: collection.chunkSize || '-'
},
...(webSelector
? [
{
label: t('common:core.dataset.collection.metadata.Web page selector'),
value: webSelector
}
]
: []),
{
...(collection.tags
? [
{
label: t('dataset:collection_tags'),
value: collection.tags?.join(', ') || '-'
}
]
: [])
}
];
}, [collection, t]);
return (
<MyBox isLoading={isLoading} w={'100%'} h={'100%'} p={6}>
<Box fontSize={'md'} pb={4}>
{t('common:core.dataset.collection.metadata.metadata')}
</Box>
<Flex mb={4} wordBreak={'break-all'} fontSize={'sm'}>
<Box color={'myGray.500'} flex={'0 0 70px'}>
{t('common:core.dataset.collection.id')}:
</Box>
<Box>{collection?._id}</Box>
</Flex>
{metadataList.map(
(item, i) =>
item.label &&
item.value && (
<Flex key={i} alignItems={'center'} mb={4} wordBreak={'break-all'} fontSize={'sm'}>
<Box color={'myGray.500'} flex={'0 0 70px'}>
{item.label}
</Box>
<Box>{item.value}</Box>
</Flex>
)
)}
{collection?.sourceId && (
<Button variant={'whitePrimary'} onClick={readSource}>
<Flex py={2} px={3}>
<MyIcon name="visible" w={'1rem'} mr={'0.38rem'} />
<Box>{t('common:core.dataset.collection.metadata.read source')}</Box>
</Flex>
</Button>
)}
</MyBox>
);
};
export default React.memo(MetaDataCard);

221
projects/app/src/pageComponents/dataset/detail/NavBar.tsx Normal file
View File

@@ -0,0 +1,221 @@
import React, { useCallback } from 'react';
import { useTranslation } from 'next-i18next';
import { Box, Flex, IconButton, useTheme, Progress } from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRouter } from 'next/router';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import LightRowTabs from '@fastgpt/web/components/common/Tabs/LightRowTabs';
import { useI18n } from '@/web/context/I18n';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import MyPopover from '@fastgpt/web/components/common/MyPopover';
import ParentPaths from '@/components/common/ParentPaths';
export enum TabEnum {
dataCard = 'dataCard',
collectionCard = 'collectionCard',
test = 'test',
info = 'info',
import = 'import'
}
const NavBar = ({ currentTab }: { currentTab: TabEnum }) => {
const theme = useTheme();
const { t } = useTranslation();
const { datasetT } = useI18n();
const router = useRouter();
const query = router.query;
const { isPc } = useSystem();
const { datasetDetail, vectorTrainingMap, agentTrainingMap, rebuildingCount, paths } =
useContextSelector(DatasetPageContext, (v) => v);
const tabList = [
{
label: t('common:core.dataset.Collection'),
value: TabEnum.collectionCard
},
{ label: t('common:core.dataset.test.Search Test'), value: TabEnum.test },
...(datasetDetail.permission.hasManagePer && !isPc
? [{ label: t('common:common.Config'), value: TabEnum.info }]
: [])
];
const setCurrentTab = useCallback(
(tab: TabEnum) => {
router.replace({
query: {
datasetId: query.datasetId,
currentTab: tab
}
});
},
[query, router]
);
return (
<>
{isPc ? (
<Flex
pb={2}
pt={3}
px={4}
justify={'space-between'}
borderBottom={currentTab === TabEnum.dataCard ? 'none' : theme.borders.base}
borderColor={'myGray.200'}
position={'relative'}
>
{currentTab === TabEnum.dataCard ? (
<>
<Flex
alignItems={'center'}
cursor={'pointer'}
py={'0.38rem'}
px={2}
ml={0}
borderRadius={'md'}
_hover={{ bg: 'myGray.05' }}
fontSize={'sm'}
fontWeight={500}
onClick={() => {
router.replace({
query: {
datasetId: router.query.datasetId,
parentId: router.query.parentId,
currentTab: TabEnum.collectionCard
}
});
}}
>
<IconButton
p={2}
mr={2}
border={'1px solid'}
borderColor={'myGray.200'}
boxShadow={'1'}
icon={<MyIcon name={'common/arrowLeft'} w={'16px'} color={'myGray.500'} />}
bg={'white'}
size={'xsSquare'}
borderRadius={'50%'}
aria-label={''}
_hover={'none'}
/>
<Box fontWeight={500} color={'myGray.600'} fontSize={'sm'}>
{datasetDetail.name}
</Box>
</Flex>
</>
) : (
<Flex py={'0.38rem'} px={2} h={10} ml={0.5}>
<ParentPaths
paths={paths}
onClick={(e) => {
router.push(`/dataset/list?parentId=${e}`);
}}
/>
</Flex>
)}
<Box position={'absolute'} left={'50%'} transform={'translateX(-50%)'}>
<LightRowTabs<TabEnum>
px={4}
py={1}
visibility={currentTab === TabEnum.dataCard ? 'hidden' : 'visible'}
flex={1}
mx={'auto'}
w={'100%'}
list={tabList}
value={currentTab}
activeColor="primary.700"
onChange={setCurrentTab}
inlineStyles={{
fontSize: '1rem',
lineHeight: '1.5rem',
fontWeight: 500,
border: 'none',
_hover: {
bg: 'myGray.05'
},
borderRadius: '6px'
}}
/>
</Box>
{/* 训练情况hover弹窗 */}
<MyPopover
placement="bottom-end"
visibility={currentTab === TabEnum.collectionCard ? 'visible' : 'hidden'}
trigger="hover"
Trigger={
<Flex
visibility={currentTab === TabEnum.collectionCard ? 'visible' : 'hidden'}
alignItems={'center'}
justifyContent={'center'}
p={2}
borderRadius={'md'}
_hover={{
bg: 'myGray.05'
}}
>
<MyIcon name={'common/monitor'} w={'18px'} h={'18px'} color={'myGray.500'} />
<Box color={'myGray.600'} ml={1.5} fontWeight={500} userSelect={'none'}>
{t('common:core.dataset.training.tag')}
</Box>
</Flex>
}
>
{({ onClose }) => (
<Box p={6}>
{rebuildingCount > 0 && (
<Box mb={3}>
<Box fontSize={'sm'}>
{datasetT('rebuilding_index_count', { count: rebuildingCount })}
</Box>
</Box>
)}
<Box mb={3}>
<Box fontSize={'sm'} pb={1}>
{t('common:core.dataset.training.Agent queue')}({agentTrainingMap.tip})
</Box>
<Progress
value={100}
size={'xs'}
colorScheme={agentTrainingMap.colorSchema}
borderRadius={'md'}
isAnimated
hasStripe
/>
</Box>
<Box>
<Box fontSize={'sm'} pb={1}>
{t('common:core.dataset.training.Vector queue')}({vectorTrainingMap.tip})
</Box>
<Progress
value={100}
size={'xs'}
colorScheme={vectorTrainingMap.colorSchema}
borderRadius={'md'}
isAnimated
hasStripe
/>
</Box>
</Box>
)}
</MyPopover>
</Flex>
) : (
<Box mb={2}>
<LightRowTabs<TabEnum>
m={'auto'}
w={'full'}
size={'sm'}
list={tabList}
value={currentTab}
onChange={setCurrentTab}
/>
</Box>
)}
</>
);
};
export default NavBar;

458
projects/app/src/pageComponents/dataset/detail/Test.tsx Normal file
View File

@@ -0,0 +1,458 @@
import React, { useEffect, useMemo, useState } from 'react';
import { Box, Textarea, Button, Flex, useTheme, useDisclosure } from '@chakra-ui/react';
import { useSearchTestStore, SearchTestStoreItemType } from '@/web/core/dataset/store/searchTest';
import { postSearchText } from '@/web/core/dataset/api';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { formatTimeToChatTime } from '@fastgpt/global/common/string/time';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { customAlphabet } from 'nanoid';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useTranslation } from 'next-i18next';
import { SearchTestResponse } from '@/global/core/dataset/api';
import {
DatasetSearchModeEnum,
DatasetSearchModeMap
} from '@fastgpt/global/core/dataset/constants';
import dynamic from 'next/dynamic';
import { useForm } from 'react-hook-form';
import MySelect from '@fastgpt/web/components/common/MySelect';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { fileDownload } from '@/web/common/file/utils';
import QuoteItem from '@/components/core/dataset/QuoteItem';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import SearchParamsTip from '@/components/core/dataset/SearchParamsTip';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
const DatasetParamsModal = dynamic(() => import('@/components/core/app/DatasetParamsModal'));
type FormType = {
inputText: string;
searchParams: {
searchMode: `${DatasetSearchModeEnum}`;
similarity?: number;
limit?: number;
usingReRank?: boolean;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
};
};
const Test = ({ datasetId }: { datasetId: string }) => {
const { t } = useTranslation();
const { toast } = useToast();
const { defaultModels } = useSystemStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { pushDatasetTestItem } = useSearchTestStore();
const [inputType, setInputType] = useState<'text' | 'file'>('text');
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
const [refresh, setRefresh] = useState(false);
const [isFocus, setIsFocus] = useState(false);
const { File, onOpen } = useSelectFile({
fileType: '.csv',
multiple: false
});
const [selectFile, setSelectFile] = useState<File>();
const { getValues, setValue, register, handleSubmit } = useForm<FormType>({
defaultValues: {
inputText: '',
searchParams: {
searchMode: DatasetSearchModeEnum.embedding,
usingReRank: false,
limit: 5000,
similarity: 0,
datasetSearchUsingExtensionQuery: false,
datasetSearchExtensionModel: defaultModels.llm?.model,
datasetSearchExtensionBg: ''
}
}
});
const searchModeData = DatasetSearchModeMap[getValues(`searchParams.searchMode`)];
const {
isOpen: isOpenSelectMode,
onOpen: onOpenSelectMode,
onClose: onCloseSelectMode
} = useDisclosure();
const { runAsync: onTextTest, loading: textTestIsLoading } = useRequest2(
({ inputText, searchParams }: FormType) =>
postSearchText({ datasetId, text: inputText.trim(), ...searchParams }),
{
onSuccess(res: SearchTestResponse) {
if (!res || res.list.length === 0) {
return toast({
status: 'warning',
title: t('common:dataset.test.noResult')
});
}
const testItem: SearchTestStoreItemType = {
id: nanoid(),
datasetId,
text: getValues('inputText').trim(),
time: new Date(),
results: res.list,
duration: res.duration,
searchMode: res.searchMode,
usingReRank: res.usingReRank,
limit: res.limit,
similarity: res.similarity,
queryExtensionModel: res.queryExtensionModel
};
pushDatasetTestItem(testItem);
setDatasetTestItem(testItem);
},
onError(err) {
toast({
title: getErrText(err),
status: 'error'
});
}
}
);
const onSelectFile = async (files: File[]) => {
const file = files[0];
if (!file) return;
setSelectFile(file);
};
useEffect(() => {
setDatasetTestItem(undefined);
}, [datasetId]);
return (
<Box h={'100%'} display={['block', 'flex']}>
{/* left */}
<Box
h={['auto', '100%']}
display={['block', 'flex']}
flexDirection={'column'}
flex={1}
maxW={'500px'}
py={4}
>
<Box
border={'2px solid'}
p={3}
mx={4}
borderRadius={'md'}
{...(isFocus
? {
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)'
}
: {
borderColor: 'primary.300'
})}
>
{/* header */}
<Flex alignItems={'center'} justifyContent={'space-between'}>
<MySelect<'text' | 'file'>
size={'sm'}
w={'150px'}
list={[
{
label: (
<Flex alignItems={'center'}>
<MyIcon mr={2} name={'text'} w={'14px'} color={'primary.600'} />
<Box fontSize={'sm'} fontWeight={'bold'} flex={1}>
{t('common:core.dataset.test.Test Text')}
</Box>
</Flex>
),
value: 'text'
}
// {
// label: (
// <Flex alignItems={'center'}>
// <MyIcon mr={2} name={'file/csv'} w={'14px'} color={'primary.600'} />
// <Box fontSize={'sm'} fontWeight={'bold'} flex={1}>
// {t('common:core.dataset.test.Batch test')}
// </Box>
// </Flex>
// ),
// value: 'file'
// }
]}
value={inputType}
onchange={(e) => setInputType(e)}
/>
<Button
variant={'whitePrimary'}
leftIcon={<MyIcon name={searchModeData.icon as any} w={'14px'} />}
size={'sm'}
onClick={onOpenSelectMode}
>
{t(searchModeData.title as any)}
</Button>
</Flex>
<Box h={'180px'}>
{inputType === 'text' && (
<Textarea
h={'100%'}
resize={'none'}
variant={'unstyled'}
maxLength={datasetDetail.vectorModel?.maxToken}
placeholder={t('common:core.dataset.test.Test Text Placeholder')}
onFocus={() => setIsFocus(true)}
{...register('inputText', {
required: true,
onBlur: () => {
setIsFocus(false);
}
})}
/>
)}
{inputType === 'file' && (
<Box pt={5}>
<Flex
p={3}
borderRadius={'md'}
borderWidth={'1px'}
borderColor={'borderColor.base'}
borderStyle={'dashed'}
bg={'white'}
cursor={'pointer'}
justifyContent={'center'}
_hover={{
bg: 'primary.100',
borderColor: 'primary.500',
borderStyle: 'solid'
}}
onClick={onOpen}
>
<MyIcon mr={2} name={'file/csv'} w={'24px'} />
<Box>
{selectFile
? selectFile.name
: t('common:core.dataset.test.Batch test Placeholder')}
</Box>
</Flex>
<Box mt={3} fontSize={'sm'}>
{t('common:info.csv_message')}
<Box
as={'span'}
color={'primary.600'}
cursor={'pointer'}
onClick={() => {
fileDownload({
text: `"问题"\n"问题1"\n"问题2"\n"问题3"`,
type: 'text/csv',
filename: 'Test Template'
});
}}
>
{t('common:info.csv_download')}
</Box>
</Box>
</Box>
)}
</Box>
<Flex justifyContent={'flex-end'}>
<Button
size={'sm'}
isLoading={textTestIsLoading}
isDisabled={inputType === 'file' && !selectFile}
onClick={() => {
if (inputType === 'text') {
handleSubmit((data) => onTextTest(data))();
} else {
// handleSubmit((data) => onFileTest(data))();
}
}}
>
{t('common:core.dataset.test.Test')}
</Button>
</Flex>
</Box>
<Box mt={5} px={4} overflow={'overlay'} display={['none', 'block']}>
<TestHistories
datasetId={datasetId}
datasetTestItem={datasetTestItem}
setDatasetTestItem={setDatasetTestItem}
/>
</Box>
</Box>
{/* result show */}
<Box p={4} h={['auto', '100%']} overflow={'overlay'} flex={'1 0 0'} bg={'white'}>
<TestResults datasetTestItem={datasetTestItem} />
</Box>
{isOpenSelectMode && (
<DatasetParamsModal
{...getValues('searchParams')}
maxTokens={20000}
onClose={onCloseSelectMode}
onSuccess={(e) => {
setValue('searchParams', {
...getValues('searchParams'),
...e
});
setRefresh((state) => !state);
}}
/>
)}
<File onSelect={onSelectFile} />
</Box>
);
};
export default React.memo(Test);
const TestHistories = React.memo(function TestHistories({
datasetId,
datasetTestItem,
setDatasetTestItem
}: {
datasetId: string;
datasetTestItem?: SearchTestStoreItemType;
setDatasetTestItem: React.Dispatch<React.SetStateAction<SearchTestStoreItemType | undefined>>;
}) {
const { t } = useTranslation();
const { datasetTestList, delDatasetTestItemById } = useSearchTestStore();
const testHistories = useMemo(
() => datasetTestList.filter((item) => item.datasetId === datasetId),
[datasetId, datasetTestList]
);
return (
<>
<Flex alignItems={'center'} color={'myGray.900'}>
<MyIcon mr={2} name={'history'} w={'18px'} h={'18px'} color={'myGray.900'} />
<Box fontSize={'md'}>{t('common:core.dataset.test.test history')}</Box>
</Flex>
<Box mt={2}>
{testHistories.map((item) => (
<Flex
key={item.id}
py={2}
px={3}
alignItems={'center'}
borderColor={'borderColor.low'}
borderWidth={'1px'}
borderRadius={'md'}
_notLast={{
mb: 2
}}
_hover={{
borderColor: 'primary.300',
boxShadow: '1',
'& .delete': {
display: 'block'
},
'& .time': {
display: 'none'
}
}}
cursor={'pointer'}
fontSize={'sm'}
{...(item.id === datasetTestItem?.id && {
bg: 'primary.50'
})}
onClick={() => setDatasetTestItem(item)}
>
<Box flex={'0 0 auto'} mr={2}>
{DatasetSearchModeMap[item.searchMode] ? (
<Flex alignItems={'center'} fontWeight={'500'} color={'myGray.500'}>
<MyIcon
name={DatasetSearchModeMap[item.searchMode].icon as any}
w={'12px'}
mr={'1px'}
/>
{t(DatasetSearchModeMap[item.searchMode].title as any)}
</Flex>
) : (
'-'
)}
</Box>
<Box flex={1} mr={2} wordBreak={'break-all'} fontWeight={'400'}>
{item.text}
</Box>
<Box className="time" flex={'0 0 auto'} fontSize={'xs'} color={'myGray.500'}>
{t(formatTimeToChatTime(item.time) as any).replace('#', ':')}
</Box>
<MyTooltip label={t('common:core.dataset.test.delete test history')}>
<Box className="delete" display={'none'} w={'0.8rem'} h={'0.8rem'} ml={1}>
<MyIcon
name={'delete'}
w={'0.8rem'}
_hover={{ color: 'red.600' }}
onClick={(e) => {
e.stopPropagation();
delDatasetTestItemById(item.id);
datasetTestItem?.id === item.id && setDatasetTestItem(undefined);
}}
/>
</Box>
</MyTooltip>
</Flex>
))}
</Box>
</>
);
});
const TestResults = React.memo(function TestResults({
datasetTestItem
}: {
datasetTestItem?: SearchTestStoreItemType;
}) {
const { t } = useTranslation();
const theme = useTheme();
return (
<>
{!datasetTestItem?.results || datasetTestItem.results.length === 0 ? (
<EmptyTip text={t('common:core.dataset.test.test result placeholder')} mt={[10, '20vh']} />
) : (
<>
<Flex fontSize={'md'} color={'myGray.900'} alignItems={'center'}>
<MyIcon name={'common/paramsLight'} w={'18px'} mr={2} />
{t('common:core.dataset.test.Test params')}
</Flex>
<Box mt={3}>
<SearchParamsTip
searchMode={datasetTestItem.searchMode}
similarity={datasetTestItem.similarity}
limit={datasetTestItem.limit}
usingReRank={datasetTestItem.usingReRank}
datasetSearchUsingExtensionQuery={!!datasetTestItem.queryExtensionModel}
queryExtensionModel={datasetTestItem.queryExtensionModel}
/>
</Box>
<Flex mt={5} mb={3} alignItems={'center'}>
<Flex fontSize={'md'} color={'myGray.900'} alignItems={'center'}>
<MyIcon name={'common/resultLight'} w={'18px'} mr={2} />
{t('common:core.dataset.test.Test Result')}
</Flex>
<QuestionTip ml={1} label={t('common:core.dataset.test.test result tip')} />
<Box ml={2}>({datasetTestItem.duration})</Box>
</Flex>
<Box mt={1} gap={4}>
{datasetTestItem?.results.map((item, index) => (
<Box key={item.id} p={3} borderRadius={'lg'} bg={'myGray.100'} _notLast={{ mb: 2 }}>
<QuoteItem quoteItem={item} canViewSource />
</Box>
))}
</Box>
</>
)}
</>
);
});

11
projects/app/src/pageComponents/dataset/detail/styles.module.scss Normal file
View File

@@ -0,0 +1,11 @@
.scrollbar {
&::-webkit-scrollbar-thumb {
background: var(--chakra-colors-myGray-150) !important;
transition: background 1s;
margin-left: 5px;
}
&::-webkit-scrollbar-thumb:hover {
background: var(--chakra-colors-myGray-250) !important;
}
}

267
projects/app/src/pageComponents/dataset/list/CreateModal.tsx Normal file
View File

@@ -0,0 +1,267 @@
import React, { useMemo } from 'react';
import { Box, Flex, Button, ModalFooter, ModalBody, Input, HStack } from '@chakra-ui/react';
import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
import { useForm } from 'react-hook-form';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { useRouter } from 'next/router';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import Avatar from '@fastgpt/web/components/common/Avatar';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { postCreateDataset } from '@/web/core/dataset/api';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import { useTranslation } from 'next-i18next';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import AIModelSelector from '@/components/Select/AIModelSelector';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import ComplianceTip from '@/components/common/ComplianceTip/index';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { getDocPath } from '@/web/common/system/doc';
import { datasetTypeCourseMap } from '@/web/core/dataset/constants';
import ApiDatasetForm from '../ApiDatasetForm';
import { getWebDefaultModel } from '@/web/common/system/utils';
export type CreateDatasetType =
| DatasetTypeEnum.dataset
| DatasetTypeEnum.apiDataset
| DatasetTypeEnum.websiteDataset
| DatasetTypeEnum.feishu
| DatasetTypeEnum.yuque;
const CreateModal = ({
onClose,
parentId,
type
}: {
onClose: () => void;
parentId?: string;
type: CreateDatasetType;
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const router = useRouter();
const { defaultModels, embeddingModelList, datasetModelList } = useSystemStore();
const { isPc } = useSystem();
const datasetTypeMap = useMemo(() => {
return {
[DatasetTypeEnum.dataset]: {
name: t('dataset:common_dataset'),
icon: 'core/dataset/commonDatasetColor'
},
[DatasetTypeEnum.apiDataset]: {
name: t('dataset:api_file'),
icon: 'core/dataset/externalDatasetColor'
},
[DatasetTypeEnum.websiteDataset]: {
name: t('dataset:website_dataset'),
icon: 'core/dataset/websiteDatasetColor'
},
[DatasetTypeEnum.feishu]: {
name: t('dataset:feishu_dataset'),
icon: 'core/dataset/feishuDatasetColor'
},
[DatasetTypeEnum.yuque]: {
name: t('dataset:yuque_dataset'),
icon: 'core/dataset/yuqueDatasetColor'
}
};
}, [t]);
const filterNotHiddenVectorModelList = embeddingModelList.filter((item) => !item.hidden);
const form = useForm<CreateDatasetParams>({
defaultValues: {
parentId,
type: type || DatasetTypeEnum.dataset,
avatar: datasetTypeMap[type].icon,
name: '',
intro: '',
vectorModel: defaultModels.embedding?.model,
agentModel: getWebDefaultModel(datasetModelList)?.model
}
});
const { register, setValue, handleSubmit, watch } = form;
const avatar = watch('avatar');
const vectorModel = watch('vectorModel');
const agentModel = watch('agentModel');
const {
File,
onOpen: onOpenSelectFile,
onSelectImage
} = useSelectFile({
fileType: 'image/*',
multiple: false
});
/* create a new kb and router to it */
const { run: onclickCreate, loading: creating } = useRequest2(
async (data: CreateDatasetParams) => await postCreateDataset(data),
{
successToast: t('common:common.Create Success'),
errorToast: t('common:common.Create Failed'),
onSuccess(id) {
router.push(`/dataset/detail?datasetId=${id}`);
}
}
);
return (
<MyModal
title={
<Flex alignItems={'center'} ml={-3}>
<Avatar
w={'20px'}
h={'20px'}
borderRadius={'xs'}
src={datasetTypeMap[type].icon}
pr={'10px'}
/>
{t('common:core.dataset.Create dataset', { name: datasetTypeMap[type].name })}
</Flex>
}
isOpen
onClose={onClose}
isCentered={!isPc}
w={'490px'}
>
<ModalBody py={6} px={9}>
<Box>
<Flex justify={'space-between'}>
<Box color={'myGray.900'} fontWeight={500} fontSize={'sm'}>
{t('common:common.Set Name')}
</Box>
{datasetTypeCourseMap[type] && (
<Flex
as={'span'}
alignItems={'center'}
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() => window.open(getDocPath(datasetTypeCourseMap[type]), '_blank')}
>
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}
</Flex>
)}
</Flex>
<Flex mt={'12px'} alignItems={'center'}>
<MyTooltip label={t('common:common.avatar.Select Avatar')}>
<Avatar
flexShrink={0}
src={avatar}
w={['28px', '32px']}
h={['28px', '32px']}
cursor={'pointer'}
borderRadius={'md'}
onClick={onOpenSelectFile}
/>
</MyTooltip>
<Input
ml={3}
flex={1}
autoFocus
bg={'myWhite.600'}
placeholder={t('common:common.Name')}
maxLength={30}
{...register('name', {
required: true
})}
/>
</Flex>
</Box>
<Flex
mt={6}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
>
<HStack
spacing={1}
alignItems={'center'}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
>
<Box>{t('common:core.ai.model.Vector Model')}</Box>
<QuestionTip label={t('common:core.dataset.embedding model tip')} />
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={vectorModel}
list={filterNotHiddenVectorModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
setValue('vectorModel' as const, e);
}}
/>
</Box>
</Flex>
<Flex
mt={6}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
>
<HStack
spacing={1}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
>
<Box>{t('common:core.ai.model.Dataset Agent Model')}</Box>
<QuestionTip label={t('dataset:file_model_function_tip')} />
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={agentModel}
list={datasetModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
setValue('agentModel' as const, e);
}}
/>
</Box>
</Flex>
{/* @ts-ignore */}
<ApiDatasetForm type={type} form={form} />
</ModalBody>
<ModalFooter px={9}>
<Button variant={'whiteBase'} mr={3} onClick={onClose}>
{t('common:common.Close')}
</Button>
<Button isLoading={creating} onClick={handleSubmit((data) => onclickCreate(data))}>
{t('common:common.Confirm Create')}
</Button>
</ModalFooter>
<ComplianceTip pb={6} pt={0} px={9} type={'dataset'} />
<File
onSelect={(e) =>
onSelectImage(e, {
maxH: 300,
maxW: 300,
callback: (e) => setValue('avatar', e)
})
}
/>
</MyModal>
);
};
export default CreateModal;

459
projects/app/src/pageComponents/dataset/list/List.tsx Normal file
View File

@@ -0,0 +1,459 @@
import React, { useMemo, useRef, useState } from 'react';
import { postChangeOwner, resumeInheritPer } from '@/web/core/dataset/api';
import { Box, Flex, Grid, HStack } from '@chakra-ui/react';
import { DatasetTypeEnum, DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
import MyMenu from '@fastgpt/web/components/common/MyMenu';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useRouter } from 'next/router';
import PermissionIconText from '@/components/support/permission/IconText';
import Avatar from '@fastgpt/web/components/common/Avatar';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { checkTeamExportDatasetLimit } from '@/web/support/user/team/api';
import { downloadFetch } from '@/web/common/system/utils';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import dynamic from 'next/dynamic';
import { useContextSelector } from 'use-context-selector';
import { DatasetsContext } from '../../../pages/dataset/list/context';
import { DatasetPermissionList } from '@fastgpt/global/support/permission/dataset/constant';
import ConfigPerModal from '@/components/support/permission/ConfigPerModal';
import {
deleteDatasetCollaborators,
getCollaboratorList,
postUpdateDatasetCollaborators
} from '@/web/core/dataset/api/collaborator';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
import { useFolderDrag } from '@/components/common/folder/useFolderDrag';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { useTranslation } from 'next-i18next';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import SideTag from './SideTag';
import { getModelProvider } from '@fastgpt/global/core/ai/provider';
import UserBox from '@fastgpt/web/components/common/UserBox';
const EditResourceModal = dynamic(() => import('@/components/common/Modal/EditResourceModal'));
function List() {
const { setLoading } = useSystemStore();
const { isPc } = useSystem();
const { t } = useTranslation();
const {
loadMyDatasets,
setMoveDatasetId,
refetchPaths,
editedDataset,
setEditedDataset,
onDelDataset,
onUpdateDataset,
myDatasets,
folderDetail
} = useContextSelector(DatasetsContext, (v) => v);
const [editPerDatasetIndex, setEditPerDatasetIndex] = useState<number>();
const router = useRouter();
const { parentId = null } = router.query as { parentId?: string | null };
const parentDataset = useMemo(
() => myDatasets.find((item) => String(item._id) === parentId),
[parentId, myDatasets]
);
const { openConfirm: openMoveConfirm, ConfirmModal: MoveConfirmModal } = useConfirm({
type: 'common',
title: t('common:move.confirm'),
content: t('dataset:move.hint')
});
const { runAsync: updateDataset } = useRequest2(onUpdateDataset);
const { getBoxProps } = useFolderDrag({
activeStyles: {
borderColor: 'primary.600'
},
onDrop: (dragId: string, targetId: string) => {
openMoveConfirm(() =>
updateDataset({
id: dragId,
parentId: targetId
})
)();
}
});
const editPerDataset = useMemo(
() => (editPerDatasetIndex !== undefined ? myDatasets[editPerDatasetIndex] : undefined),
[editPerDatasetIndex, myDatasets]
);
const { mutate: exportDataset } = useRequest({
mutationFn: async (dataset: DatasetItemType) => {
setLoading(true);
await checkTeamExportDatasetLimit(dataset._id);
await downloadFetch({
url: `/api/core/dataset/exportAll?datasetId=${dataset._id}`,
filename: `${dataset.name}.csv`
});
},
onSettled() {
setLoading(false);
},
successToast: t('common:core.dataset.Start export'),
errorToast: t('common:dataset.Export Dataset Limit Error')
});
const DeleteTipsMap = useRef({
[DatasetTypeEnum.folder]: t('common:dataset.deleteFolderTips'),
[DatasetTypeEnum.dataset]: t('common:core.dataset.Delete Confirm'),
[DatasetTypeEnum.websiteDataset]: t('common:core.dataset.Delete Confirm'),
[DatasetTypeEnum.externalFile]: t('common:core.dataset.Delete Confirm')
});
const formatDatasets = useMemo(
() =>
myDatasets.map((item) => {
return {
...item,
label: DatasetTypeMap[item.type]?.label,
icon: DatasetTypeMap[item.type]?.icon
};
}),
[myDatasets]
);
const { openConfirm, ConfirmModal } = useConfirm({
type: 'delete'
});
const onClickDeleteDataset = (id: string) => {
openConfirm(
() =>
onDelDataset(id).then(() => {
refetchPaths();
loadMyDatasets();
}),
undefined,
DeleteTipsMap.current[DatasetTypeEnum.dataset]
)();
};
return (
<>
{formatDatasets.length > 0 && (
<Grid
py={4}
gridTemplateColumns={
folderDetail
? ['1fr', 'repeat(2,1fr)', 'repeat(2,1fr)', 'repeat(3,1fr)']
: ['1fr', 'repeat(2,1fr)', 'repeat(3,1fr)', 'repeat(3,1fr)', 'repeat(4,1fr)']
}
gridGap={5}
alignItems={'stretch'}
>
{formatDatasets.map((dataset, index) => {
const vectorModelAvatar = getModelProvider(dataset.vectorModel.provider)?.avatar;
return (
<MyTooltip
key={dataset._id}
label={
<Flex flexDirection={'column'} alignItems={'center'}>
<Box fontSize={'xs'} color={'myGray.500'}>
{dataset.type === DatasetTypeEnum.folder
? t('common:common.folder.Open folder')
: t('common:common.folder.open_dataset')}
</Box>
</Flex>
}
>
<MyBox
display={'flex'}
flexDirection={'column'}
lineHeight={1.5}
h="100%"
pt={5}
pb={3}
px={5}
cursor={'pointer'}
borderWidth={1.5}
border={'base'}
boxShadow={'2'}
bg={'white'}
borderRadius={'lg'}
position={'relative'}
minH={'150px'}
{...getBoxProps({
dataId: dataset._id,
isFolder: dataset.type === DatasetTypeEnum.folder
})}
_hover={{
borderColor: 'primary.300',
boxShadow: '1.5',
'& .delete': {
display: 'block'
},
'& .more': {
display: 'flex'
},
'& .time': {
display: ['flex', 'none']
}
}}
onClick={() => {
if (dataset.type === DatasetTypeEnum.folder) {
router.push({
pathname: '/dataset/list',
query: {
parentId: dataset._id
}
});
} else {
router.push({
pathname: '/dataset/detail',
query: {
datasetId: dataset._id
}
});
}
}}
>
<HStack>
<Avatar src={dataset.avatar} borderRadius={6} w={'28px'} />
<Box flex={'1 0 0'} className="textEllipsis3" color={'myGray.900'}>
{dataset.name}
</Box>
<Box mr={'-1.25rem'}>
{dataset.type !== DatasetTypeEnum.folder && (
<SideTag
type={dataset.type}
py={0.5}
px={2}
borderLeftRadius={'sm'}
borderRightRadius={0}
/>
)}
</Box>
</HStack>
<Box
flex={1}
className={'textEllipsis3'}
py={3}
wordBreak={'break-all'}
fontSize={'xs'}
color={'myGray.500'}
>
{dataset.intro ||
(dataset.type === DatasetTypeEnum.folder
? t('common:core.dataset.Folder placeholder')
: t('common:core.dataset.Intro Placeholder'))}
</Box>
<Flex
h={'24px'}
alignItems={'center'}
justifyContent={'space-between'}
fontSize={'sm'}
fontWeight={500}
color={'myGray.500'}
>
<HStack spacing={3.5}>
<UserBox
sourceMember={dataset.sourceMember}
fontSize="xs"
avatarSize="1rem"
spacing={0.5}
/>
<PermissionIconText
flexShrink={0}
private={dataset.private}
iconColor="myGray.400"
color={'myGray.500'}
/>
</HStack>
<HStack>
{isPc && dataset.type !== DatasetTypeEnum.folder && (
<HStack spacing={1} className="time">
<Avatar src={vectorModelAvatar} w={'0.85rem'} />
<Box color={'myGray.500'} fontSize={'mini'}>
{dataset.vectorModel.name}
</Box>
</HStack>
)}
{(dataset.type === DatasetTypeEnum.folder
? dataset.permission.hasManagePer
: dataset.permission.hasWritePer) && (
<Box
className="more"
display={['', 'none']}
borderRadius={'md'}
_hover={{
'& .icon': {
bg: 'myGray.100'
}
}}
onClick={(e) => {
e.stopPropagation();
}}
>
<MyMenu
Button={
<Box w={'22px'} h={'22px'}>
<MyIcon
className="icon"
name={'more'}
h={'16px'}
w={'16px'}
px={1}
py={1}
borderRadius={'md'}
cursor={'pointer'}
/>
</Box>
}
menuList={[
{
children: [
{
icon: 'edit',
label: t('common:dataset.Edit Info'),
onClick: () =>
setEditedDataset({
id: dataset._id,
name: dataset.name,
intro: dataset.intro,
avatar: dataset.avatar
})
},
...((parentDataset ? parentDataset : dataset)?.permission
.hasManagePer
? [
{
icon: 'common/file/move',
label: t('common:Move'),
onClick: () => {
setMoveDatasetId(dataset._id);
}
}
]
: []),
...(dataset.permission.hasManagePer
? [
{
icon: 'key',
label: t('common:permission.Permission'),
onClick: () => setEditPerDatasetIndex(index)
}
]
: [])
]
},
...(dataset.type != DatasetTypeEnum.folder
? [
{
children: [
{
icon: 'export',
label: t('common:Export'),
onClick: () => {
exportDataset(dataset);
}
}
]
}
]
: []),
...(dataset.permission.hasManagePer
? [
{
children: [
{
icon: 'delete',
label: t('common:common.Delete'),
type: 'danger' as 'danger',
onClick: () => onClickDeleteDataset(dataset._id)
}
]
}
]
: [])
]}
/>
</Box>
)}
</HStack>
</Flex>
</MyBox>
</MyTooltip>
);
})}
</Grid>
)}
{myDatasets.length === 0 && (
<EmptyTip
pt={'35vh'}
text={t('common:core.dataset.Empty Dataset Tips')}
flexGrow="1"
></EmptyTip>
)}
{editedDataset && (
<EditResourceModal
{...editedDataset}
title={t('common:dataset.Edit Info')}
onClose={() => setEditedDataset(undefined)}
onEdit={async (data) => {
await onUpdateDataset({
id: editedDataset.id,
name: data.name,
intro: data.intro,
avatar: data.avatar
});
}}
/>
)}
{!!editPerDataset && (
<ConfigPerModal
onChangeOwner={(tmbId: string) =>
postChangeOwner({
datasetId: editPerDataset._id,
ownerId: tmbId
}).then(() => loadMyDatasets())
}
hasParent={!!parentId}
refetchResource={loadMyDatasets}
isInheritPermission={editPerDataset.inheritPermission}
resumeInheritPermission={() =>
resumeInheritPer(editPerDataset._id).then(() => Promise.all([loadMyDatasets()]))
}
avatar={editPerDataset.avatar}
name={editPerDataset.name}
managePer={{
permission: editPerDataset.permission,
onGetCollaboratorList: () => getCollaboratorList(editPerDataset._id),
permissionList: DatasetPermissionList,
onUpdateCollaborators: (props) =>
postUpdateDatasetCollaborators({
...props,
datasetId: editPerDataset._id
}),
onDelOneCollaborator: async (props) =>
deleteDatasetCollaborators({
...props,
datasetId: editPerDataset._id
}),
refreshDeps: [editPerDataset._id, editPerDataset.inheritPermission]
}}
onClose={() => setEditPerDatasetIndex(undefined)}
/>
)}
<ConfirmModal />
<MoveConfirmModal />
</>
);
}
export default List;

59
projects/app/src/pageComponents/dataset/list/SideTag.tsx Normal file
View File

@@ -0,0 +1,59 @@
import { Box, Flex, FlexProps } from '@chakra-ui/react';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import MyIcon from '@fastgpt/web/components/common/Icon';
import React, { useMemo } from 'react';
import { useTranslation } from 'next-i18next';
const SideTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps) => {
if (type === DatasetTypeEnum.folder) return null;
const { t } = useTranslation();
const DatasetListTypeMap = useMemo(() => {
return {
[DatasetTypeEnum.dataset]: {
icon: 'core/dataset/commonDatasetOutline',
label: t('dataset:common_dataset')
},
[DatasetTypeEnum.websiteDataset]: {
icon: 'core/dataset/websiteDatasetOutline',
label: t('dataset:website_dataset')
},
[DatasetTypeEnum.externalFile]: {
icon: 'core/dataset/externalDatasetOutline',
label: t('dataset:external_file')
},
[DatasetTypeEnum.apiDataset]: {
icon: 'core/dataset/externalDatasetOutline',
label: t('dataset:api_file')
},
[DatasetTypeEnum.feishu]: {
icon: 'core/dataset/feishuDatasetOutline',
label: t('dataset:feishu_dataset')
},
[DatasetTypeEnum.yuque]: {
icon: 'core/dataset/yuqueDatasetOutline',
label: t('dataset:yuque_dataset')
}
};
}, [t]);
const item = DatasetListTypeMap[type] || DatasetListTypeMap['dataset'];
return (
<Flex
bg={'myGray.100'}
py={0.75}
pl={'8px'}
pr={'12px'}
borderRadius={'md'}
fontSize={'xs'}
alignItems={'center'}
{...props}
>
<MyIcon name={item.icon as any} w={'0.8rem'} color={'myGray.400'} />
<Box fontSize={'mini'} ml={1}>
{item.label}
</Box>
</Flex>
);
};
export default SideTag;