gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: api dataset code

Browse Source
This commit is contained in:
archer
2025-06-03 18:31:35 +08:00
parent 2507997d20
commit e32ca8a3e9
63 changed files with 347 additions and 530 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx
View File

@@ -3,11 +3,6 @@ import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { Flex, Input, Button, ModalBody, ModalFooter, Box } from '@chakra-ui/react';
import type { UseFormReturn } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import type {
APIFileServer,
FeishuServer,
YuqueServer
} from '@fastgpt/global/core/dataset/apiDataset';
import { getApiDatasetPaths, getApiDatasetCatalog } from '@/web/core/dataset/api';
import type {
GetResourceFolderListItemResponse,
@@ -22,6 +17,7 @@ import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { FolderIcon } from '@fastgpt/global/common/file/image/constants';
import type { ApiDatasetServerType } from '@fastgpt/global/core/dataset/apiDataset/type';
const ApiDatasetForm = ({
type,
@@ -32,9 +28,7 @@ const ApiDatasetForm = ({
datasetId?: string;
form: UseFormReturn<
{
apiServer?: APIFileServer;
feishuServer?: FeishuServer;
yuqueServer?: YuqueServer;
apiDatasetServer?: ApiDatasetServerType;
},
any
>;
@@ -42,9 +36,10 @@ const ApiDatasetForm = ({
const { t } = useTranslation();
const { register, setValue, watch } = form;
const yuqueServer = watch('yuqueServer');
const feishuServer = watch('feishuServer');
const apiServer = watch('apiServer');
const apiDatasetServer = watch('apiDatasetServer');
const yuqueServer = apiDatasetServer?.yuqueServer;
const feishuServer = apiDatasetServer?.feishuServer;
const apiServer = apiDatasetServer?.apiServer;
const [pathNames, setPathNames] = useState(t('dataset:rootdirectory'));
const [
@@ -91,9 +86,7 @@ const ApiDatasetForm = ({
const path = await getApiDatasetPaths({
datasetId,
parentId,
yuqueServer,
feishuServer,
apiServer
apiDatasetServer
});
setPathNames(path);
},
@@ -108,13 +101,13 @@ const ApiDatasetForm = ({
const value = id === 'root' || id === null || id === undefined ? '' : id;
switch (type) {
case DatasetTypeEnum.yuque:
setValue('yuqueServer.basePath', value);
setValue('apiDatasetServer.yuqueServer.basePath', value);
break;
case DatasetTypeEnum.feishu:
setValue('feishuServer.folderToken', value);
setValue('apiDatasetServer.feishuServer.folderToken', value);
break;
case DatasetTypeEnum.apiDataset:
setValue('apiServer.basePath', value);
setValue('apiDatasetServer.apiServer.basePath', value);
break;
}
@@ -147,32 +140,10 @@ const ApiDatasetForm = ({
<BaseUrlSelector
selectId={yuqueServer?.basePath || apiServer?.basePath || 'root'}
server={async (e: GetResourceFolderListProps) => {
const params: GetApiDatasetCataLogProps = { parentId: e.parentId };
switch (type) {
case DatasetTypeEnum.yuque:
params.yuqueServer = {
userId: yuqueServer?.userId || '',
token: yuqueServer?.token || '',
basePath: ''
};
break;
// Currently, only Yuque is using it
case DatasetTypeEnum.feishu:
params.feishuServer = {
appId: feishuServer?.appId || '',
appSecret: feishuServer?.appSecret || '',
folderToken: feishuServer?.folderToken || ''
};
break;
case DatasetTypeEnum.apiDataset:
params.apiServer = {
baseUrl: apiServer?.baseUrl || '',
authorization: apiServer?.authorization || '',
basePath: ''
};
break;
}
const params: GetApiDatasetCataLogProps = {
parentId: e.parentId,
apiDatasetServer
};
return getApiDatasetCatalog(params);
}}
@@ -193,7 +164,7 @@ const ApiDatasetForm = ({
bg={'myWhite.600'}
placeholder={t('dataset:api_url')}
maxLength={200}
{...register('apiServer.baseUrl', { required: true })}
{...register('apiDatasetServer.apiServer.baseUrl', { required: true })}
/>
</Flex>
<Flex mt={6} alignItems={'center'}>
@@ -204,7 +175,7 @@ const ApiDatasetForm = ({
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
maxLength={2000}
{...register('apiServer.authorization')}
{...register('apiDatasetServer.apiServer.authorization')}
/>
</Flex>
{renderBaseUrlSelector()}
@@ -227,7 +198,7 @@ const ApiDatasetForm = ({
bg={'myWhite.600'}
placeholder={'App ID'}
maxLength={200}
{...register('feishuServer.appId', { required: true })}
{...register('apiDatasetServer.feishuServer.appId', { required: true })}
/>
</Flex>
<Flex mt={6}>
@@ -244,7 +215,7 @@ const ApiDatasetForm = ({
bg={'myWhite.600'}
placeholder={'App Secret'}
maxLength={200}
{...register('feishuServer.appSecret', { required: true })}
{...register('apiDatasetServer.feishuServer.appSecret', { required: true })}
/>
</Flex>
<Flex mt={6}>
@@ -261,7 +232,7 @@ const ApiDatasetForm = ({
bg={'myWhite.600'}
placeholder={'Folder Token'}
maxLength={200}
{...register('feishuServer.folderToken', { required: true })}
{...register('apiDatasetServer.feishuServer.folderToken', { required: true })}
/>
</Flex>
{/* {renderBaseUrlSelector()}
@@ -278,7 +249,7 @@ const ApiDatasetForm = ({
bg={'myWhite.600'}
placeholder={'User ID'}
maxLength={200}
{...register('yuqueServer.userId', { required: true })}
{...register('apiDatasetServer.yuqueServer.userId', { required: true })}
/>
</Flex>
<Flex mt={6} alignItems={'center'}>
@@ -289,7 +260,7 @@ const ApiDatasetForm = ({
bg={'myWhite.600'}
placeholder={'Token'}
maxLength={200}
{...register('yuqueServer.token', { required: true })}
{...register('apiDatasetServer.yuqueServer.token', { required: true })}
/>
</Flex>
{renderBaseUrlSelector()}

7
projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx
View File

@@ -17,7 +17,8 @@ import {
DatasetCollectionTypeEnum,
DatasetTypeEnum,
DatasetTypeMap,
DatasetStatusEnum
DatasetStatusEnum,
ApiDatasetTypeMap
} from '@fastgpt/global/core/dataset/constants';
import EditFolderModal, { useEditFolder } from '../../EditFolderModal';
import { TabEnum } from '../../../../pages/dataset/detail/index';
@@ -435,9 +436,7 @@ const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
/>
)}
{/* apiDataset */}
{(datasetDetail?.type === DatasetTypeEnum.apiDataset ||
datasetDetail?.type === DatasetTypeEnum.feishu ||
datasetDetail?.type === DatasetTypeEnum.yuque) && (
{datasetDetail?.type && ApiDatasetTypeMap[datasetDetail.type] && (
<Flex
px={3.5}
py={2}

2
projects/app/src/pageComponents/dataset/detail/Import/diffSource/APIDataset.tsx
View File

@@ -13,7 +13,7 @@ import { type ParentTreePathItemType } from '@fastgpt/global/common/parentFolder
import FolderPath from '@/components/common/folder/Path';
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { type APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
import { type APIFileItem } from '@fastgpt/global/core/dataset/apiDataset/type';
import SearchInput from '@fastgpt/web/components/common/Input/SearchInput';
import { useMount } from 'ahooks';

16
projects/app/src/pageComponents/dataset/detail/Info/components/EditApiServiceModal.tsx
View File

@@ -5,23 +5,17 @@ import { useTranslation } from 'next-i18next';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useForm } from 'react-hook-form';
import { useToast } from '@fastgpt/web/hooks/useToast';
import {
type APIFileServer,
type FeishuServer,
type YuqueServer
} from '@fastgpt/global/core/dataset/apiDataset';
import ApiDatasetForm from '@/pageComponents/dataset/ApiDatasetForm';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { datasetTypeCourseMap } from '@/web/core/dataset/constants';
import { getDocPath } from '@/web/common/system/doc';
import MyIcon from '@fastgpt/web/components/common/Icon';
import type { ApiDatasetServerType } from '@fastgpt/global/core/dataset/apiDataset/type';
import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
export type EditAPIDatasetInfoFormType = {
id: string;
apiServer?: APIFileServer;
yuqueServer?: YuqueServer;
feishuServer?: FeishuServer;
apiDatasetServer?: ApiDatasetServerType;
};
const EditAPIDatasetInfoModal = ({
@@ -60,7 +54,7 @@ const EditAPIDatasetInfoModal = ({
return (
<MyModal isOpen onClose={onClose} w={'450px'} iconSrc="modal/edit" title={title}>
<ModalBody>
{datasetTypeCourseMap[type] && (
{DatasetTypeMap[type]?.courseUrl && (
<Flex alignItems={'center'} justifyContent={'space-between'}>
<Box color={'myGray.900'} fontSize={'sm'} fontWeight={500}>
{t('dataset:apidataset_configuration')}
@@ -71,7 +65,7 @@ const EditAPIDatasetInfoModal = ({
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() => window.open(getDocPath(datasetTypeCourseMap[type]), '_blank')}
onClick={() => window.open(getDocPath(DatasetTypeMap[type].courseUrl!), '_blank')}
>
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}

18
projects/app/src/pageComponents/dataset/detail/Info/index.tsx
View File

@@ -311,12 +311,12 @@ const Info = ({ datasetId }: { datasetId: string }) => {
onClick={() =>
setEditedAPIDataset({
id: datasetDetail._id,
apiServer: datasetDetail.apiServer
apiDatasetServer: datasetDetail.apiDatasetServer
})
}
/>
</Flex>
<Box fontSize={'mini'}>{datasetDetail.apiServer?.baseUrl}</Box>
<Box fontSize={'mini'}>{datasetDetail.apiDatasetServer?.apiServer?.baseUrl}</Box>
</Box>
</>
)}
@@ -336,12 +336,12 @@ const Info = ({ datasetId }: { datasetId: string }) => {
onClick={() =>
setEditedAPIDataset({
id: datasetDetail._id,
yuqueServer: datasetDetail.yuqueServer
apiDatasetServer: datasetDetail.apiDatasetServer
})
}
/>
</Flex>
<Box fontSize={'mini'}>{datasetDetail.yuqueServer?.userId}</Box>
<Box fontSize={'mini'}>{datasetDetail.apiDatasetServer?.yuqueServer?.userId}</Box>
</Box>
</>
)}
@@ -361,12 +361,14 @@ const Info = ({ datasetId }: { datasetId: string }) => {
onClick={() =>
setEditedAPIDataset({
id: datasetDetail._id,
feishuServer: datasetDetail.feishuServer
apiDatasetServer: datasetDetail.apiDatasetServer
})
}
/>
</Flex>
<Box fontSize={'mini'}>{datasetDetail.feishuServer?.folderToken}</Box>
<Box fontSize={'mini'}>
{datasetDetail.apiDatasetServer?.feishuServer?.folderToken}
</Box>
</Box>
</>
)}
@@ -435,9 +437,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
onEdit={(data) =>
updateDataset({
id: datasetId,
apiServer: data.apiServer,
yuqueServer: data.yuqueServer,
feishuServer: data.feishuServer
apiDatasetServer: data.apiDatasetServer
})
}
/>

38
projects/app/src/pageComponents/dataset/list/CreateModal.tsx
View File

@@ -11,14 +11,13 @@ import MyModal from '@fastgpt/web/components/common/MyModal';
import { postCreateDataset } from '@/web/core/dataset/api';
import type { CreateDatasetParams } from '@/global/core/dataset/api.d';
import { useTranslation } from 'next-i18next';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { DatasetTypeEnum, DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
import AIModelSelector from '@/components/Select/AIModelSelector';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import ComplianceTip from '@/components/common/ComplianceTip/index';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { getDocPath } from '@/web/common/system/doc';
import { datasetTypeCourseMap } from '@/web/core/dataset/constants';
import ApiDatasetForm from '../ApiDatasetForm';
import { getWebDefaultEmbeddingModel, getWebDefaultLLMModel } from '@/web/common/system/utils';
@@ -43,31 +42,6 @@ const CreateModal = ({
const { defaultModels, embeddingModelList, datasetModelList, getVlmModelList } = useSystemStore();
const { isPc } = useSystem();
const datasetTypeMap = useMemo(() => {
return {
[DatasetTypeEnum.dataset]: {
name: t('dataset:common_dataset'),
icon: 'core/dataset/commonDatasetColor'
},
[DatasetTypeEnum.websiteDataset]: {
name: t('dataset:website_dataset'),
icon: 'core/dataset/websiteDatasetColor'
},
[DatasetTypeEnum.apiDataset]: {
name: t('dataset:api_file'),
icon: 'core/dataset/externalDatasetColor'
},
[DatasetTypeEnum.feishu]: {
name: t('dataset:feishu_dataset'),
icon: 'core/dataset/feishuDatasetColor'
},
[DatasetTypeEnum.yuque]: {
name: t('dataset:yuque_dataset'),
icon: 'core/dataset/yuqueDatasetColor'
}
};
}, [t]);
const filterNotHiddenVectorModelList = embeddingModelList.filter((item) => !item.hidden);
const vllmModelList = useMemo(() => getVlmModelList(), [getVlmModelList]);
@@ -76,7 +50,7 @@ const CreateModal = ({
defaultValues: {
parentId,
type: type || DatasetTypeEnum.dataset,
avatar: datasetTypeMap[type].icon,
avatar: DatasetTypeMap[type].icon,
name: '',
intro: '',
vectorModel:
@@ -121,10 +95,10 @@ const CreateModal = ({
w={'20px'}
h={'20px'}
borderRadius={'xs'}
src={datasetTypeMap[type].icon}
src={DatasetTypeMap[type].icon}
pr={'10px'}
/>
{t('common:core.dataset.Create dataset', { name: datasetTypeMap[type].name })}
{t('common:core.dataset.Create dataset', { name: t(DatasetTypeMap[type].label) })}
</Flex>
}
isOpen
@@ -138,14 +112,14 @@ const CreateModal = ({
<Box color={'myGray.900'} fontWeight={500} fontSize={'sm'}>
{t('common:input_name')}
</Box>
{datasetTypeCourseMap[type] && (
{DatasetTypeMap[type]?.courseUrl && (
<Flex
as={'span'}
alignItems={'center'}
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() => window.open(getDocPath(datasetTypeCourseMap[type]), '_blank')}
onClick={() => window.open(getDocPath(DatasetTypeMap[type].courseUrl!), '_blank')}
>
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}

37
projects/app/src/pageComponents/dataset/list/SideTag.tsx
View File

@@ -1,41 +1,14 @@
import { Box, Flex, type FlexProps } from '@chakra-ui/react';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { DatasetTypeEnum, DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
import MyIcon from '@fastgpt/web/components/common/Icon';
import React, { useMemo } from 'react';
import React from 'react';
import { useTranslation } from 'next-i18next';
const SideTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps) => {
if (type === DatasetTypeEnum.folder) return null;
const { t } = useTranslation();
const DatasetListTypeMap = useMemo(() => {
return {
[DatasetTypeEnum.dataset]: {
icon: 'core/dataset/commonDatasetOutline',
label: t('dataset:common_dataset')
},
[DatasetTypeEnum.websiteDataset]: {
icon: 'core/dataset/websiteDatasetOutline',
label: t('dataset:website_dataset')
},
[DatasetTypeEnum.externalFile]: {
icon: 'core/dataset/externalDatasetOutline',
label: t('dataset:external_file')
},
[DatasetTypeEnum.apiDataset]: {
icon: 'core/dataset/externalDatasetOutline',
label: t('dataset:api_file')
},
[DatasetTypeEnum.feishu]: {
icon: 'core/dataset/feishuDatasetOutline',
label: t('dataset:feishu_dataset')
},
[DatasetTypeEnum.yuque]: {
icon: 'core/dataset/yuqueDatasetOutline',
label: t('dataset:yuque_dataset')
}
};
}, [t]);
const item = DatasetListTypeMap[type] || DatasetListTypeMap['dataset'];
const item = DatasetTypeMap[type] || DatasetTypeMap['dataset'];
return (
<Flex
@@ -50,7 +23,7 @@ const SideTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps)
>
<MyIcon name={item.icon as any} w={'0.8rem'} color={'myGray.400'} />
<Box fontSize={'mini'} ml={1}>
{item.label}
{t(item.label)}
</Box>
</Flex>
);