V4.8.16 dev (#3431)

* feat: add feishu & yuque dataset (#3379)

* feat: add feishu & yuque dataset

* fix ts

* fix ts

* move type position

* fix

* fix: merge interface

* fix

* feat: dingtalk sso support (#3408)

* fix: optional sso state

* feat: dingtalk bot

* feat: dingtalk sso login

* chore: move i18n to user namespace

* feat: dingtalk bot integration (#3415)

* feat: dingtalk bot integration

* docs: config dingtalk bot

* feat:sear XNG服务 (#3413)

* feat:sear XNG服务

* 补充了courseUrl

* 添加了官方文档

* 错误时返回情况修正了一下

* Tracks (#3420)

* feat: node intro

* feat: add domain track

* dingding sso login

* perf: api dataset code and add doc

* feat: tracks

* feat: searXNG plugins

* fix: ts

* feat: delete node tracks (#3423)

* fix: dingtalk bot GET verification (#3424)

* 4.8.16 test: fix: plugin inputs render;fix: ui offset (#3426)

* fix: ui offset

* perf: dingding talk

* fix: plugin inputs render

* feat: menu all folder (#3429)

* fix: recall code

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
This commit is contained in:
Archer
2024-12-18 19:30:19 +08:00
committed by GitHub
parent 82871be054
commit bd79e7701f
154 changed files with 2519 additions and 300 deletions

View File

@@ -0,0 +1,166 @@
import React from 'react';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { Flex, Input } from '@chakra-ui/react';
import { UseFormReturn } from 'react-hook-form';
import { useTranslation } from 'next-i18next';
import type {
APIFileServer,
FeishuServer,
YuqueServer
} from '@fastgpt/global/core/dataset/apiDataset';
const ApiDatasetForm = ({
type,
form
}: {
type: `${DatasetTypeEnum}`;
form: UseFormReturn<
{
apiServer?: APIFileServer;
feishuServer?: FeishuServer;
yuqueServer?: YuqueServer;
},
any
>;
}) => {
const { t } = useTranslation();
const { register } = form;
return (
<>
{type === DatasetTypeEnum.apiDataset && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
{t('dataset:api_url')}
</Flex>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:api_url')}
maxLength={200}
{...register('apiServer.baseUrl', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
Authorization
</Flex>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
maxLength={200}
{...register('apiServer.authorization')}
/>
</Flex>
</>
)}
{type === DatasetTypeEnum.feishu && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
App ID
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'App ID'}
maxLength={200}
{...register('feishuServer.appId', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
App Secret
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'App Secret'}
maxLength={200}
{...register('feishuServer.appSecret', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
Folder Token
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'Folder Token'}
maxLength={200}
{...register('feishuServer.folderToken', { required: true })}
/>
</Flex>
</>
)}
{type === DatasetTypeEnum.yuque && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
User ID
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'Token'}
maxLength={200}
{...register('yuqueServer.userId', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
Token
</Flex>
<Input
bg={'myWhite.600'}
placeholder={'Token'}
maxLength={200}
{...register('yuqueServer.token', { required: true })}
/>
</Flex>
</>
)}
</>
);
};
export default ApiDatasetForm;

View File

@@ -386,7 +386,9 @@ const Header = ({}: {}) => {
/>
)}
{/* apiDataset */}
{datasetDetail?.type === DatasetTypeEnum.apiDataset && (
{(datasetDetail?.type === DatasetTypeEnum.apiDataset ||
datasetDetail?.type === DatasetTypeEnum.feishu ||
datasetDetail?.type === DatasetTypeEnum.yuque) && (
<Flex
px={3.5}
py={2}

View File

@@ -177,8 +177,7 @@ const Upload = () => {
router.replace({
query: {
datasetId: datasetDetail._id,
currentTab: retrainNewCollectionId.current ? TabEnum.dataCard : TabEnum.collectionCard,
collectionId: retrainNewCollectionId.current
currentTab: TabEnum.collectionCard
}
});
},

View File

@@ -49,23 +49,26 @@ const CustomAPIFileInput = () => {
parentId: '',
parentName: ''
});
const [parentUuid, setParentUuid] = useState<string>('');
const [paths, setPaths] = useState<ParentTreePathItemType[]>([]);
const [searchKey, setSearchKey] = useState('');
const { data: fileList = [], loading } = useRequest2(
async () =>
datasetDetail?.apiServer
? getApiDatasetFileList({
datasetId: datasetDetail._id,
parentId: parent?.parentId,
searchKey: searchKey
})
: [],
async () => {
return getApiDatasetFileList({
datasetId: datasetDetail._id,
parentId: parent?.parentId,
searchKey: searchKey
});
},
{
refreshDeps: [datasetDetail._id, datasetDetail.apiServer, parent, searchKey],
throttleWait: 500,
manual: false
}
);
const { data: existIdList = [] } = useRequest2(
() => getApiDatasetFileListExistId({ datasetId: datasetDetail._id }),
{
@@ -90,6 +93,7 @@ const CustomAPIFileInput = () => {
datasetId: datasetDetail._id,
parentId: file?.id
});
const subFiles = await getFilesRecursively(folderFiles);
allFiles.push(...subFiles);
} else {
@@ -125,6 +129,7 @@ const CustomAPIFileInput = () => {
const handleItemClick = useCallback(
(item: APIFileItem) => {
if (item.type === 'folder') {
setPaths((state) => [...state, { parentId: item.id, parentName: item.name }]);
return setParent({
parentId: item.id,
parentName: item.name
@@ -138,7 +143,7 @@ const CustomAPIFileInput = () => {
setSelectFiles((state) => [...state, item]);
}
},
[selectFiles, setSelectFiles]
[selectFiles]
);
const handleSelectAll = useCallback(() => {
@@ -151,8 +156,6 @@ const CustomAPIFileInput = () => {
}
}, [fileList, selectFiles]);
const paths = useMemo(() => [parent || { parentId: '', parentName: '' }], [parent]);
return (
<MyBox isLoading={loading} position="relative" h="full">
<Flex flexDirection={'column'} h="full">
@@ -160,21 +163,21 @@ const CustomAPIFileInput = () => {
<FolderPath
paths={paths}
onClick={(parentId) => {
if (parentId !== parent?.parentId) {
setParent({
parentId,
parentName: ''
});
}
const index = paths.findIndex((item) => item.parentId === parentId);
setParent(paths[index]);
setPaths(paths.slice(0, index + 1));
}}
/>
<Box w={'240px'}>
<SearchInput
value={searchKey}
onChange={(e) => setSearchKey(e.target.value)}
placeholder={t('common:core.workflow.template.Search')}
/>
</Box>
{datasetDetail.apiServer && (
<Box w={'240px'}>
<SearchInput
value={searchKey}
onChange={(e) => setSearchKey(e.target.value)}
placeholder={t('common:core.workflow.template.Search')}
/>
</Box>
)}
</Flex>
<Box flex={1} overflowY="auto" mb={16}>
<Box ml={2} mt={3}>

View File

@@ -1,15 +1,23 @@
import React from 'react';
import { ModalFooter, ModalBody, Input, Button, Flex } from '@chakra-ui/react';
import { ModalFooter, ModalBody, Button, Flex } from '@chakra-ui/react';
import MyModal from '@fastgpt/web/components/common/MyModal/index';
import { useTranslation } from 'next-i18next';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useForm } from 'react-hook-form';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { APIFileServer } from '@fastgpt/global/core/dataset/apiDataset';
import { APIFileServer, FeishuServer, YuqueServer } from '@fastgpt/global/core/dataset/apiDataset';
import ApiDatasetForm from '@/pages/dataset/component/ApiDatasetForm';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { datasetTypeCourseMap } from '@/web/core/dataset/constants';
import { getDocPath } from '@/web/common/system/doc';
import MyIcon from '@fastgpt/web/components/common/Icon';
export type EditAPIDatasetInfoFormType = {
id: string;
apiServer?: APIFileServer;
yuqueServer?: YuqueServer;
feishuServer?: FeishuServer;
};
const EditAPIDatasetInfoModal = ({
@@ -24,7 +32,11 @@ const EditAPIDatasetInfoModal = ({
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const { register, handleSubmit } = useForm<EditAPIDatasetInfoFormType>({
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const type = datasetDetail.type;
const form = useForm<EditAPIDatasetInfoFormType>({
defaultValues: defaultForm
});
@@ -44,43 +56,24 @@ const EditAPIDatasetInfoModal = ({
return (
<MyModal isOpen onClose={onClose} w={'450px'} iconSrc="modal/edit" title={title}>
<ModalBody>
<Flex>
{datasetTypeCourseMap[type] && (
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
justifyContent={'flex-end'}
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() => window.open(getDocPath(datasetTypeCourseMap[type]), '_blank')}
>
{t('dataset:api_url')}
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}
</Flex>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:api_url')}
maxLength={200}
{...register('apiServer.baseUrl', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
Authorization
</Flex>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
maxLength={200}
{...register('apiServer.authorization')}
/>
</Flex>
)}
{/* @ts-ignore */}
<ApiDatasetForm type={type} form={form} />
</ModalBody>
<ModalFooter>
<Button isLoading={loading} onClick={handleSubmit(onSave)} px={6}>
<Button isLoading={loading} onClick={form.handleSubmit(onSave)} px={6}>
{t('common:common.Confirm')}
</Button>
</ModalFooter>

View File

@@ -29,13 +29,12 @@ import {
} from '@/web/core/dataset/api/collaborator';
import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag';
import dynamic from 'next/dynamic';
import EditAPIDatasetInfoModal, {
EditAPIDatasetInfoFormType
} from './components/EditApiServiceModal';
import type { EditAPIDatasetInfoFormType } from './components/EditApiServiceModal';
import { EditResourceInfoFormType } from '@/components/common/Modal/EditResourceModal';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
const EditResourceModal = dynamic(() => import('@/components/common/Modal/EditResourceModal'));
const EditAPIDatasetInfoModal = dynamic(() => import('./components/EditApiServiceModal'));
const Info = ({ datasetId }: { datasetId: string }) => {
const { t } = useTranslation();
@@ -326,6 +325,56 @@ const Info = ({ datasetId }: { datasetId: string }) => {
</Box>
</>
)}
{datasetDetail.type === DatasetTypeEnum.yuque && (
<>
<Box w={'100%'} alignItems={'center'} pt={4}>
<Flex justifyContent={'space-between'} mb={1}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:yuque_dataset_config')}
</FormLabel>
<MyIcon
name={'edit'}
w={'14px'}
_hover={{ color: 'primary.600' }}
cursor={'pointer'}
onClick={() =>
setEditedAPIDataset({
id: datasetDetail._id,
yuqueServer: datasetDetail.yuqueServer
})
}
/>
</Flex>
<Box fontSize={'mini'}>{datasetDetail.yuqueServer?.userId}</Box>
</Box>
</>
)}
{datasetDetail.type === DatasetTypeEnum.feishu && (
<>
<Box w={'100%'} alignItems={'center'} pt={4}>
<Flex justifyContent={'space-between'} mb={1}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:feishu_dataset_config')}
</FormLabel>
<MyIcon
name={'edit'}
w={'14px'}
_hover={{ color: 'primary.600' }}
cursor={'pointer'}
onClick={() =>
setEditedAPIDataset({
id: datasetDetail._id,
feishuServer: datasetDetail.feishuServer
})
}
/>
</Flex>
<Box fontSize={'mini'}>{datasetDetail.feishuServer?.folderToken}</Box>
</Box>
</>
)}
</Box>
{datasetDetail.permission.hasManagePer && (
@@ -384,12 +433,14 @@ const Info = ({ datasetId }: { datasetId: string }) => {
{editedAPIDataset && (
<EditAPIDatasetInfoModal
{...editedAPIDataset}
title={t('common:dataset.Edit API Service')}
title={t('dataset:edit_dataset_config')}
onClose={() => setEditedAPIDataset(undefined)}
onEdit={(data) =>
updateDataset({
id: datasetId,
apiServer: data.apiServer
apiServer: data.apiServer,
yuqueServer: data.yuqueServer,
feishuServer: data.feishuServer
})
}
/>

View File

@@ -22,11 +22,15 @@ import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import ComplianceTip from '@/components/common/ComplianceTip/index';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { getDocPath } from '@/web/common/system/doc';
import { datasetTypeCourseMap } from '@/web/core/dataset/constants';
import ApiDatasetForm from '../../component/ApiDatasetForm';
export type CreateDatasetType =
| DatasetTypeEnum.dataset
| DatasetTypeEnum.apiDataset
| DatasetTypeEnum.websiteDataset;
| DatasetTypeEnum.websiteDataset
| DatasetTypeEnum.feishu
| DatasetTypeEnum.yuque;
const CreateModal = ({
onClose,
@@ -43,35 +47,45 @@ const CreateModal = ({
const { vectorModelList, datasetModelList } = useSystemStore();
const { isPc } = useSystem();
const databaseNameMap = useMemo(() => {
const datasetTypeMap = useMemo(() => {
return {
[DatasetTypeEnum.dataset]: t('dataset:common_dataset'),
[DatasetTypeEnum.apiDataset]: t('dataset:api_file'),
[DatasetTypeEnum.websiteDataset]: t('dataset:website_dataset')
[DatasetTypeEnum.dataset]: {
name: t('dataset:common_dataset'),
icon: 'core/dataset/commonDatasetColor'
},
[DatasetTypeEnum.apiDataset]: {
name: t('dataset:api_file'),
icon: 'core/dataset/externalDatasetColor'
},
[DatasetTypeEnum.websiteDataset]: {
name: t('dataset:website_dataset'),
icon: 'core/dataset/websiteDatasetColor'
},
[DatasetTypeEnum.feishu]: {
name: t('dataset:feishu_dataset'),
icon: 'core/dataset/feishuDatasetColor'
},
[DatasetTypeEnum.yuque]: {
name: t('dataset:yuque_dataset'),
icon: 'core/dataset/yuqueDatasetColor'
}
};
}, [t]);
const iconMap = useMemo(() => {
return {
[DatasetTypeEnum.dataset]: 'core/dataset/commonDatasetColor',
[DatasetTypeEnum.apiDataset]: 'core/dataset/externalDatasetColor',
[DatasetTypeEnum.websiteDataset]: 'core/dataset/websiteDatasetColor'
};
}, []);
const filterNotHiddenVectorModelList = vectorModelList.filter((item) => !item.hidden);
const { register, setValue, handleSubmit, watch } = useForm<CreateDatasetParams>({
const form = useForm<CreateDatasetParams>({
defaultValues: {
parentId,
type: type || DatasetTypeEnum.dataset,
avatar: iconMap[type] || 'core/dataset/commonDatasetColor',
avatar: datasetTypeMap[type].icon,
name: '',
intro: '',
vectorModel: filterNotHiddenVectorModelList[0].model,
agentModel: datasetModelList[0].model
}
});
const { register, setValue, handleSubmit, watch } = form;
const avatar = watch('avatar');
const vectorModel = watch('vectorModel');
const agentModel = watch('agentModel');
@@ -119,8 +133,14 @@ const CreateModal = ({
<MyModal
title={
<Flex alignItems={'center'} ml={-3}>
<Avatar w={'20px'} h={'20px'} borderRadius={'xs'} src={iconMap[type]} pr={'10px'} />
{t('common:core.dataset.Create dataset', { name: databaseNameMap[type] })}
<Avatar
w={'20px'}
h={'20px'}
borderRadius={'xs'}
src={datasetTypeMap[type].icon}
pr={'10px'}
/>
{t('common:core.dataset.Create dataset', { name: datasetTypeMap[type].name })}
</Flex>
}
isOpen
@@ -134,16 +154,14 @@ const CreateModal = ({
<Box color={'myGray.900'} fontWeight={500} fontSize={'sm'}>
{t('common:common.Set Name')}
</Box>
{type === DatasetTypeEnum.apiDataset && (
{datasetTypeCourseMap[type] && (
<Flex
as={'span'}
alignItems={'center'}
color={'primary.600'}
fontSize={'sm'}
cursor={'pointer'}
onClick={() =>
window.open(getDocPath('/docs/guide/knowledge_base/api_dataset/'), '_blank')
}
onClick={() => window.open(getDocPath(datasetTypeCourseMap[type]), '_blank')}
>
<MyIcon name={'book'} w={4} mr={0.5} />
{t('common:Instructions')}
@@ -242,44 +260,8 @@ const CreateModal = ({
</Box>
</Flex>
)}
{type === DatasetTypeEnum.apiDataset && (
<>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
{t('dataset:api_url')}
</Flex>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:api_url')}
maxLength={200}
{...register('apiServer.baseUrl', { required: true })}
/>
</Flex>
<Flex mt={6}>
<Flex
alignItems={'center'}
flex={['', '0 0 110px']}
color={'myGray.900'}
fontWeight={500}
fontSize={'sm'}
>
Authorization
</Flex>
<Input
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
maxLength={200}
{...register('apiServer.authorization')}
/>
</Flex>
</>
)}
{/* @ts-ignore */}
<ApiDatasetForm type={type} form={form} />
</ModalBody>
<ModalFooter px={9}>

View File

@@ -24,6 +24,14 @@ const SideTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps)
[DatasetTypeEnum.apiDataset]: {
icon: 'core/dataset/externalDatasetOutline',
label: t('dataset:api_file')
},
[DatasetTypeEnum.feishu]: {
icon: 'core/dataset/feishuDatasetOutline',
label: t('dataset:feishu_dataset')
},
[DatasetTypeEnum.yuque]: {
icon: 'core/dataset/yuqueDatasetOutline',
label: t('dataset:yuque_dataset')
}
};
}, [t]);

View File

@@ -64,7 +64,10 @@ const Dataset = () => {
const onSelectDatasetType = useCallback(
(e: CreateDatasetType) => {
if (!feConfigs?.isPlus && e === DatasetTypeEnum.websiteDataset) {
if (
!feConfigs?.isPlus &&
[DatasetTypeEnum.websiteDataset, DatasetTypeEnum.feishu, DatasetTypeEnum.yuque].includes(e)
) {
return toast({
status: 'warning',
title: t('common:common.system.Commercial version function')
@@ -168,6 +171,18 @@ const Dataset = () => {
label: t('dataset:website_dataset'),
description: t('dataset:website_dataset_desc'),
onClick: () => onSelectDatasetType(DatasetTypeEnum.websiteDataset)
},
{
icon: 'core/dataset/feishuDatasetColor',
label: t('dataset:feishu_dataset'),
description: t('dataset:feishu_dataset_desc'),
onClick: () => onSelectDatasetType(DatasetTypeEnum.feishu)
},
{
icon: 'core/dataset/yuqueDatasetColor',
label: t('dataset:yuque_dataset'),
description: t('dataset:yuque_dataset_desc'),
onClick: () => onSelectDatasetType(DatasetTypeEnum.yuque)
}
]
},