gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V4.9.4 feature (#4470)

Browse Source 
* Training status (#4424)

* dataset data training state (#4311)

* dataset data training state

* fix

* fix ts

* fix

* fix api format

* fix

* fix

* perf: count training

* format

* fix: dataset training state (#4417)

* fix

* add test

* fix

* fix

* fix test

* fix test

* perf: training count

* count

* loading status

---------

Co-authored-by: heheer <heheer@sealos.io>

* doc

* website sync feature (#4429)

* perf: introduce BullMQ for website sync (#4403)

* perf: introduce BullMQ for website sync

* feat: new redis module

* fix: remove graceful shutdown

* perf: improve UI in dataset detail

- Updated the "change" icon SVG file.
- Modified i18n strings.
- Added new i18n string "immediate_sync".
- Improved UI in dataset detail page, including button icons and
background colors.

* refactor: Add chunkSettings to DatasetSchema

* perf: website sync ux

* env template

* fix: clean up website dataset when updating chunk settings (#4420)

* perf: check setting updated

* perf: worker currency

* feat: init script for website sync refactor (#4425)

* website feature doc

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

* pro migration (#4388) (#4433)

* pro migration

* reuse customPdfParseType

Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>

* perf: remove loading ui

* feat: config chat file expired time

* Redis cache (#4436)

* perf: add Redis cache for vector counting (#4432)

* feat: cache

* perf: get cache key

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

* perf: mobile voice input (#4437)

* update:Mobile voice interaction (#4362)

* Add files via upload

* Add files via upload

* Update ollama.md

* Update ollama.md

* Add files via upload

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update constants.ts

* Add files via upload

* Update ChatInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update useSpeech.ts

* Update ChatInput.tsx

* Add files via upload

* Update common.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update common.json

* Update common.json

* Update common.json

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update VoiceInput.tsx

* speech ui

* 优化语音输入组件，调整输入框显示逻辑，修复语音输入遮罩层样式，更新画布背景透明度，增强用户交互体验。 (#4435)

* perf: mobil voice input

---------

Co-authored-by: dreamer6680 <1468683855@qq.com>

* Test completion v2 (#4438)

* add v2 completions (#4364)

* add v2 completions

* completion config

* config version

* fix

* frontend

* doc

* fix

* fix: completions v2 api

---------

Co-authored-by: heheer <heheer@sealos.io>

* package

* Test mongo log (#4443)

* feat: mongodb-log (#4426)

* perf: mongo log

* feat: completions stop reasoner

* mongo db log

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>

* update doc

* Update doc

* fix external var ui (#4444)

* action

* fix: ts (#4458)

* preview doc action

add docs preview permission

update preview action

udpate action

* update doc (#4460)

* update preview action

* update doc

* remove

* update

* schema

* update mq export;perf: redis cache  (#4465)

* perf: redis cache

* update mq export

* perf: website sync error tip

* add error worker

* website sync ui (#4466)

* Updated the dynamic display of the voice input pop-up (#4469)

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* fix: voice input

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>
Co-authored-by: dreamer6680 <1468683855@qq.com>
Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
This commit is contained in:
Archer
2025-04-08 12:05:04 +08:00
committed by GitHub
parent 5839325f77
commit f642c9603b
151 changed files with 5434 additions and 1354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx
View File

@@ -1,19 +1,18 @@
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { Dispatch, ReactNode, SetStateAction, useEffect, useState } from 'react';
import { Dispatch, ReactNode, SetStateAction, useState } from 'react';
import { useTranslation } from 'next-i18next';
import { createContext, useContextSelector } from 'use-context-selector';
import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { useRequest } from '@fastgpt/web/hooks/useRequest';
import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useDisclosure } from '@chakra-ui/react';
import { checkTeamWebSyncLimit } from '@/web/support/user/team/api';
import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api';
import { getDatasetCollections, postWebsiteSync } from '@/web/core/dataset/api';
import dynamic from 'next/dynamic';
import { usePagination } from '@fastgpt/web/hooks/usePagination';
import { DatasetCollectionsListItemType } from '@/global/core/dataset/type';
import { useRouter } from 'next/router';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { WebsiteConfigFormType } from './WebsiteConfig';
const WebSiteConfigModal = dynamic(() => import('./WebsiteConfig'));
@@ -66,7 +65,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
const router = useRouter();
const { parentId = '' } = router.query as { parentId: string };
const { datasetDetail, datasetId, updateDataset } = useContextSelector(
const { datasetDetail, datasetId, updateDataset, loadDatasetDetail } = useContextSelector(
DatasetPageContext,
(v) => v
);
@@ -75,30 +74,32 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
const { openConfirm: openWebSyncConfirm, ConfirmModal: ConfirmWebSyncModal } = useConfirm({
content: t('dataset:start_sync_website_tip')
});
const syncWebsite = async () => {
await checkTeamWebSyncLimit();
postWebsiteSync({ datasetId: datasetId }).then(() => {
loadDatasetDetail(datasetId);
});
};
const {
isOpen: isOpenWebsiteModal,
onOpen: onOpenWebsiteModal,
onClose: onCloseWebsiteModal
} = useDisclosure();
const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({
mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => {
onCloseWebsiteModal();
await checkTeamWebSyncLimit();
const { runAsync: onUpdateDatasetWebsiteConfig } = useRequest2(
async (websiteConfig: WebsiteConfigFormType) => {
await updateDataset({
id: datasetId,
websiteConfig,
status: DatasetStatusEnum.syncing
websiteConfig: websiteConfig.websiteConfig,
chunkSettings: websiteConfig.chunkSettings
});
const billId = await postCreateTrainingUsage({
name: t('common:core.dataset.training.Website Sync'),
datasetId: datasetId
});
await postWebsiteSync({ datasetId: datasetId, billId });
return;
await syncWebsite();
},
errorToast: t('common:common.Update Failed')
});
{
onSuccess() {
onCloseWebsiteModal();
}
}
);
// collection list
const [searchText, setSearchText] = useState('');
@@ -124,7 +125,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
});
const contextValue: CollectionPageContextType = {
openWebSyncConfirm: openWebSyncConfirm(onUpdateDatasetWebsiteConfig),
openWebSyncConfirm: openWebSyncConfirm(syncWebsite),
onOpenWebsiteModal,
searchText,
@@ -149,10 +150,6 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
<WebSiteConfigModal
onClose={onCloseWebsiteModal}
onSuccess={onUpdateDatasetWebsiteConfig}
defaultValue={{
url: datasetDetail?.websiteConfig?.url,
selector: datasetDetail?.websiteConfig?.selector
}}
/>
)}
<ConfirmWebSyncModal />

3
projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx
View File

@@ -25,6 +25,9 @@ const EmptyCollectionTip = () => {
{datasetDetail.status === DatasetStatusEnum.syncing && (
<>{t('common:core.dataset.status.syncing')}</>
)}
{datasetDetail.status === DatasetStatusEnum.waiting && (
<>{t('common:core.dataset.status.waiting')}</>
)}
{datasetDetail.status === DatasetStatusEnum.active && (
<>
{!datasetDetail?.websiteConfig?.url ? (

147
projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx
View File

@@ -1,35 +1,23 @@
import React from 'react';
import {
Box,
Flex,
MenuButton,
Button,
Link,
useTheme,
useDisclosure,
HStack
} from '@chakra-ui/react';
import { Box, Flex, MenuButton, Button, Link, useDisclosure, HStack } from '@chakra-ui/react';
import {
getDatasetCollectionPathById,
postDatasetCollection,
putDatasetCollectionById
} from '@/web/core/dataset/api';
import { useQuery } from '@tanstack/react-query';
import { useTranslation } from 'next-i18next';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyInput from '@/components/MyInput';
import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { useRouter } from 'next/router';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyMenu from '@fastgpt/web/components/common/MyMenu';
import { useEditTitle } from '@/web/common/hooks/useEditTitle';
import {
DatasetCollectionTypeEnum,
TrainingModeEnum,
DatasetTypeEnum,
DatasetTypeMap,
DatasetStatusEnum,
DatasetCollectionDataProcessModeEnum
DatasetStatusEnum
} from '@fastgpt/global/core/dataset/constants';
import EditFolderModal, { useEditFolder } from '../../EditFolderModal';
import { TabEnum } from '../../../../pages/dataset/detail/index';
@@ -43,26 +31,36 @@ import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContex
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import HeaderTagPopOver from './HeaderTagPopOver';
import MyBox from '@fastgpt/web/components/common/MyBox';
import Icon from '@fastgpt/web/components/common/Icon';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector'));
const Header = ({}: {}) => {
const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
const { t } = useTranslation();
const theme = useTheme();
const { feConfigs } = useSystemStore();
const { isPc } = useSystem();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const router = useRouter();
const { parentId = '' } = router.query as { parentId: string };
const { isPc } = useSystem();
const { searchText, setSearchText, total, getData, pageNum, onOpenWebsiteModal } =
useContextSelector(CollectionPageContext, (v) => v);
const {
searchText,
setSearchText,
total,
getData,
pageNum,
onOpenWebsiteModal,
openWebSyncConfirm
} = useContextSelector(CollectionPageContext, (v) => v);
const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
getDatasetCollectionPathById(parentId)
);
const { data: paths = [] } = useRequest2(() => getDatasetCollectionPathById(parentId), {
refreshDeps: [parentId],
manual: false
});
const { editFolderData, setEditFolderData } = useEditFolder();
const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } =
@@ -72,13 +70,14 @@ const Header = ({}: {}) => {
canEmpty: false
});
// Import collection
const {
isOpen: isOpenFileSourceSelector,
onOpen: onOpenFileSourceSelector,
onClose: onCloseFileSourceSelector
} = useDisclosure();
const { runAsync: onCreateCollection, loading: onCreating } = useRequest2(
const { runAsync: onCreateCollection } = useRequest2(
async ({ name, type }: { name: string; type: DatasetCollectionTypeEnum }) => {
const id = await postDatasetCollection({
parentId,
@@ -100,7 +99,7 @@ const Header = ({}: {}) => {
const isWebSite = datasetDetail?.type === DatasetTypeEnum.websiteDataset;
return (
<MyBox isLoading={onCreating} display={['block', 'flex']} alignItems={'center'} gap={2}>
<MyBox display={['block', 'flex']} alignItems={'center'} gap={2}>
<HStack flex={1}>
<Box flex={1} fontWeight={'500'} color={'myGray.900'} whiteSpace={'nowrap'}>
<ParentPath
@@ -121,13 +120,15 @@ const Header = ({}: {}) => {
{!isWebSite && <MyIcon name="common/list" mr={2} w={'20px'} color={'black'} />}
{t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel as any)}({total})
</Flex>
{/* Website sync */}
{datasetDetail?.websiteConfig?.url && (
<Flex fontSize={'mini'}>
{t('common:core.dataset.website.Base Url')}:
<Box>{t('common:core.dataset.website.Base Url')}:</Box>
<Link
className="textEllipsis"
maxW={'300px'}
href={datasetDetail.websiteConfig.url}
target="_blank"
mr={2}
color={'blue.700'}
>
{datasetDetail.websiteConfig.url}
@@ -171,12 +172,14 @@ const Header = ({}: {}) => {
)}
{/* Tag */}
{datasetDetail.permission.hasWritePer && feConfigs?.isPlus && <HeaderTagPopOver />}
{datasetDetail.type !== DatasetTypeEnum.websiteDataset &&
datasetDetail.permission.hasWritePer &&
feConfigs?.isPlus && <HeaderTagPopOver />}
</HStack>
{/* diff collection button */}
{datasetDetail.permission.hasWritePer && (
<Box textAlign={'end'} mt={[3, 0]}>
<Box mt={[3, 0]}>
{datasetDetail?.type === DatasetTypeEnum.dataset && (
<MyMenu
offset={[0, 5]}
@@ -233,9 +236,8 @@ const Header = ({}: {}) => {
onClick: () => {
onOpenCreateVirtualFileModal({
defaultVal: '',
onSuccess: (name) => {
onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual });
}
onSuccess: (name) =>
onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual })
});
}
},
@@ -272,35 +274,70 @@ const Header = ({}: {}) => {
{datasetDetail?.type === DatasetTypeEnum.websiteDataset && (
<>
{datasetDetail?.websiteConfig?.url ? (
<Flex alignItems={'center'}>
<>
{datasetDetail.status === DatasetStatusEnum.active && (
<Button onClick={onOpenWebsiteModal}>{t('common:common.Config')}</Button>
<HStack gap={2}>
<Button
onClick={onOpenWebsiteModal}
leftIcon={<Icon name="change" w={'1rem'} />}
>
{t('dataset:params_config')}
</Button>
{!hasTrainingData && (
<Button
variant={'whitePrimary'}
onClick={openWebSyncConfirm}
leftIcon={<Icon name="common/confirm/restoreTip" w={'1rem'} />}
>
{t('dataset:immediate_sync')}
</Button>
)}
</HStack>
)}
{datasetDetail.status === DatasetStatusEnum.syncing && (
<Flex
ml={3}
alignItems={'center'}
<MyTag
colorSchema="purple"
showDot
px={3}
py={1}
borderRadius="md"
border={theme.borders.base}
h={'36px'}
DotStyles={{
w: '8px',
h: '8px',
animation: 'zoomStopIcon 0.5s infinite alternate'
}}
>
<Box
animation={'zoomStopIcon 0.5s infinite alternate'}
bg={'myGray.700'}
w="8px"
h="8px"
borderRadius={'50%'}
mt={'1px'}
></Box>
<Box ml={2} color={'myGray.600'}>
{t('common:core.dataset.status.syncing')}
</Box>
</Flex>
{t('common:core.dataset.status.syncing')}
</MyTag>
)}
</Flex>
{datasetDetail.status === DatasetStatusEnum.waiting && (
<MyTag
colorSchema="gray"
showDot
px={3}
h={'36px'}
DotStyles={{
w: '8px',
h: '8px',
animation: 'zoomStopIcon 0.5s infinite alternate'
}}
>
{t('common:core.dataset.status.waiting')}
</MyTag>
)}
{datasetDetail.status === DatasetStatusEnum.error && (
<MyTag colorSchema="red" showDot px={3} h={'36px'}>
<HStack spacing={1}>
<Box>{t('dataset:status_error')}</Box>
<QuestionTip color={'red.500'} label={datasetDetail.errorMsg} />
</HStack>
</MyTag>
)}
</>
) : (
<Button onClick={onOpenWebsiteModal}>
<Button
onClick={onOpenWebsiteModal}
leftIcon={<Icon name="common/setting" w={'18px'} />}
>
{t('common:core.dataset.Set Website Config')}
</Button>
)}

502
projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingStates.tsx Normal file
View File

@@ -0,0 +1,502 @@
import {
Box,
Button,
Flex,
ModalBody,
Table,
TableContainer,
Tbody,
Td,
Th,
Thead,
Tr
} from '@chakra-ui/react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useTranslation } from 'next-i18next';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs';
import { useMemo, useState } from 'react';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import {
deleteTrainingData,
getDatasetCollectionTrainingDetail,
getTrainingDataDetail,
getTrainingError,
updateTrainingData
} from '@/web/core/dataset/api';
import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { getTrainingDataDetailResponse } from '@/pages/api/core/dataset/training/getTrainingDataDetail';
import MyTextarea from '@/components/common/Textarea/MyTextarea';
import { TrainingProcess } from '@/web/core/dataset/constants';
import { useForm } from 'react-hook-form';
import type { getTrainingDetailResponse } from '@/pages/api/core/dataset/collection/trainingDetail';
import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
enum TrainingStatus {
NotStart = 'NotStart',
Queued = 'Queued', // wait count>0
Running = 'Running', // wait count=0; training count>0.
Ready = 'Ready',
Error = 'Error'
}
const ProgressView = ({ trainingDetail }: { trainingDetail: getTrainingDetailResponse }) => {
const { t } = useTranslation();
const isQA = trainingDetail?.trainingType === DatasetCollectionDataProcessModeEnum.qa;
/*
状态计算
1. 暂时没有内容解析的状态
2. 完全没有训练数据时候，已就绪
3. 有训练数据，中间过程全部是进行中
*/
const statesArray = useMemo(() => {
const isReady =
Object.values(trainingDetail.queuedCounts).every((count) => count === 0) &&
Object.values(trainingDetail.trainingCounts).every((count) => count === 0) &&
Object.values(trainingDetail.errorCounts).every((count) => count === 0);
const getTrainingStatus = ({ errorCount }: { errorCount: number }) => {
if (isReady) return TrainingStatus.Ready;
if (errorCount > 0) {
return TrainingStatus.Error;
}
return TrainingStatus.Running;
};
// 只显示排队和处理中的数量
const getStatusText = (mode: TrainingModeEnum) => {
if (isReady) return;
if (trainingDetail.queuedCounts[mode] > 0) {
return t('dataset:dataset.Training_Waiting', {
count: trainingDetail.queuedCounts[mode]
});
}
if (trainingDetail.trainingCounts[mode] > 0) {
return t('dataset:dataset.Training_Count', {
count: trainingDetail.trainingCounts[mode]
});
}
return;
};
const states: {
label: string;
statusText?: string;
status: TrainingStatus;
errorCount: number;
}[] = [
// {
// label: TrainingProcess.waiting.label,
// status: TrainingStatus.Queued,
// statusText: t('dataset:dataset.Completed')
// },
{
label: t(TrainingProcess.parsing.label),
status: TrainingStatus.Ready,
errorCount: 0
},
...(isQA
? [
{
errorCount: trainingDetail.errorCounts.qa,
label: t(TrainingProcess.getQA.label),
statusText: getStatusText(TrainingModeEnum.qa),
status: getTrainingStatus({
errorCount: trainingDetail.errorCounts.qa
})
}
]
: []),
...(trainingDetail?.advancedTraining.imageIndex && !isQA
? [
{
errorCount: trainingDetail.errorCounts.image,
label: t(TrainingProcess.imageIndex.label),
statusText: getStatusText(TrainingModeEnum.image),
status: getTrainingStatus({
errorCount: trainingDetail.errorCounts.image
})
}
]
: []),
...(trainingDetail?.advancedTraining.autoIndexes && !isQA
? [
{
errorCount: trainingDetail.errorCounts.auto,
label: t(TrainingProcess.autoIndex.label),
statusText: getStatusText(TrainingModeEnum.auto),
status: getTrainingStatus({
errorCount: trainingDetail.errorCounts.auto
})
}
]
: []),
{
errorCount: trainingDetail.errorCounts.chunk,
label: t(TrainingProcess.vectorizing.label),
statusText: getStatusText(TrainingModeEnum.chunk),
status: getTrainingStatus({
errorCount: trainingDetail.errorCounts.chunk
})
},
{
errorCount: 0,
label: t('dataset:process.Is_Ready'),
status: isReady ? TrainingStatus.Ready : TrainingStatus.NotStart,
statusText: isReady
? undefined
: t('dataset:training_ready', {
count: trainingDetail.trainedCount
})
}
];
return states;
}, [trainingDetail, t, isQA]);
return (
<Flex flexDirection={'column'} gap={6}>
{statesArray.map((item, index) => (
<Flex alignItems={'center'} pl={4} key={index}>
{/* Status round */}
<Box
w={'14px'}
h={'14px'}
borderWidth={'2px'}
borderRadius={'50%'}
position={'relative'}
display={'flex'}
alignItems={'center'}
justifyContent={'center'}
{...((item.status === TrainingStatus.Running ||
item.status === TrainingStatus.Error) && {
bg: 'primary.600',
borderColor: 'primary.600',
boxShadow: '0 0 0 4px var(--Royal-Blue-100, #E1EAFF)'
})}
{...(item.status === TrainingStatus.Ready && {
bg: 'primary.600',
borderColor: 'primary.600'
})}
// Line
{...(index !== statesArray.length - 1 && {
_after: {
content: '""',
height: '59px',
width: '2px',
bgColor: 'myGray.250',
position: 'absolute',
top: '14px',
left: '4px'
}
})}
>
{item.status === TrainingStatus.Ready && (
<MyIcon name="common/check" w={3} color={'white'} />
)}
</Box>
{/* Card */}
<Flex
alignItems={'center'}
w={'full'}
bg={
item.status === TrainingStatus.Running
? 'primary.50'
: item.status === TrainingStatus.Error
? 'red.50'
: 'myGray.50'
}
py={2.5}
px={3}
ml={5}
borderRadius={'8px'}
flex={1}
h={'53px'}
>
<Box
fontSize={'14px'}
fontWeight={'medium'}
color={item.status === TrainingStatus.NotStart ? 'myGray.400' : 'myGray.900'}
mr={2}
>
{t(item.label as any)}
</Box>
{item.status === TrainingStatus.Error && (
<MyTag
showDot
type={'borderSolid'}
px={1}
fontSize={'mini'}
borderRadius={'md'}
h={5}
colorSchema={'red'}
>
{t('dataset:training.Error', { count: item.errorCount })}
</MyTag>
)}
<Box flex={1} />
{!!item.statusText && (
<Flex fontSize={'sm'} alignItems={'center'}>
{item.statusText}
</Flex>
)}
</Flex>
</Flex>
))}
</Flex>
);
};
const ErrorView = ({ datasetId, collectionId }: { datasetId: string; collectionId: string }) => {
const { t } = useTranslation();
const TrainingText = {
[TrainingModeEnum.chunk]: t('dataset:process.Vectorizing'),
[TrainingModeEnum.qa]: t('dataset:process.Get QA'),
[TrainingModeEnum.image]: t('dataset:process.Image_Index'),
[TrainingModeEnum.auto]: t('dataset:process.Auto_Index')
};
const [editChunk, setEditChunk] = useState<getTrainingDataDetailResponse>();
const {
data: errorList,
ScrollData,
isLoading,
refreshList
} = useScrollPagination(getTrainingError, {
pageSize: 15,
params: {
collectionId
},
EmptyTip: <EmptyTip />
});
const { runAsync: getData, loading: getDataLoading } = useRequest2(
(data: { datasetId: string; collectionId: string; dataId: string }) => {
return getTrainingDataDetail(data);
},
{
manual: true,
onSuccess: (data) => {
setEditChunk(data);
}
}
);
const { runAsync: deleteData, loading: deleteLoading } = useRequest2(
(data: { datasetId: string; collectionId: string; dataId: string }) => {
return deleteTrainingData(data);
},
{
manual: true,
onSuccess: () => {
refreshList();
}
}
);
const { runAsync: updateData, loading: updateLoading } = useRequest2(
(data: { datasetId: string; collectionId: string; dataId: string; q?: string; a?: string }) => {
return updateTrainingData(data);
},
{
manual: true,
onSuccess: () => {
refreshList();
setEditChunk(undefined);
}
}
);
if (editChunk) {
return (
<EditView
editChunk={editChunk}
onCancel={() => setEditChunk(undefined)}
onSave={(data) => {
updateData({
datasetId,
collectionId,
dataId: editChunk._id,
...data
});
}}
/>
);
}
return (
<ScrollData
h={'400px'}
isLoading={isLoading || updateLoading || getDataLoading || deleteLoading}
>
<TableContainer overflowY={'auto'} fontSize={'12px'}>
<Table variant={'simple'}>
<Thead>
<Tr>
<Th pr={0}>{t('dataset:dataset.Chunk_Number')}</Th>
<Th pr={0}>{t('dataset:dataset.Training_Status')}</Th>
<Th>{t('dataset:dataset.Error_Message')}</Th>
<Th>{t('dataset:dataset.Operation')}</Th>
</Tr>
</Thead>
<Tbody>
{errorList.map((item, index) => (
<Tr key={index}>
<Td>{item.chunkIndex + 1}</Td>
<Td>{TrainingText[item.mode]}</Td>
<Td maxW={50}>
<MyTooltip label={item.errorMsg}>{item.errorMsg}</MyTooltip>
</Td>
<Td>
<Flex alignItems={'center'}>
<Button
variant={'ghost'}
size={'sm'}
color={'myGray.600'}
leftIcon={<MyIcon name={'common/confirm/restoreTip'} w={4} />}
fontSize={'mini'}
onClick={() => updateData({ datasetId, collectionId, dataId: item._id })}
>
{t('dataset:dataset.ReTrain')}
</Button>
<Box w={'1px'} height={'16px'} bg={'myGray.200'} />
<Button
variant={'ghost'}
size={'sm'}
color={'myGray.600'}
leftIcon={<MyIcon name={'edit'} w={4} />}
fontSize={'mini'}
onClick={() => getData({ datasetId, collectionId, dataId: item._id })}
>
{t('dataset:dataset.Edit_Chunk')}
</Button>
<Box w={'1px'} height={'16px'} bg={'myGray.200'} />
<Button
variant={'ghost'}
size={'sm'}
color={'myGray.600'}
leftIcon={<MyIcon name={'delete'} w={4} />}
fontSize={'mini'}
onClick={() => {
deleteData({ datasetId, collectionId, dataId: item._id });
}}
>
{t('dataset:dataset.Delete_Chunk')}
</Button>
</Flex>
</Td>
</Tr>
))}
</Tbody>
</Table>
</TableContainer>
</ScrollData>
);
};
const EditView = ({
editChunk,
onCancel,
onSave
}: {
editChunk: getTrainingDataDetailResponse;
onCancel: () => void;
onSave: (data: { q: string; a?: string }) => void;
}) => {
const { t } = useTranslation();
const { register, handleSubmit } = useForm({
defaultValues: {
q: editChunk?.q || '',
a: editChunk?.a || ''
}
});
return (
<Flex flexDirection={'column'} gap={4}>
{editChunk?.a && <Box>q</Box>}
<MyTextarea {...register('q')} minH={editChunk?.a ? 200 : 400} />
{editChunk?.a && (
<>
<Box>a</Box>
<MyTextarea {...register('a')} minH={200} />
</>
)}
<Flex justifyContent={'flex-end'} gap={4}>
<Button variant={'outline'} onClick={onCancel}>
{t('common:common.Cancel')}
</Button>
<Button variant={'primary'} onClick={handleSubmit(onSave)}>
{t('dataset:dataset.ReTrain')}
</Button>
</Flex>
</Flex>
);
};
const TrainingStates = ({
datasetId,
collectionId,
defaultTab = 'states',
onClose
}: {
datasetId: string;
collectionId: string;
defaultTab?: 'states' | 'errors';
onClose: () => void;
}) => {
const { t } = useTranslation();
const [tab, setTab] = useState<typeof defaultTab>(defaultTab);
const { data: trainingDetail, loading } = useRequest2(
() => getDatasetCollectionTrainingDetail(collectionId),
{
pollingInterval: 5000,
pollingWhenHidden: false,
manual: false
}
);
const errorCounts = (Object.values(trainingDetail?.errorCounts || {}) as number[]).reduce(
(acc, count) => acc + count,
0
);
return (
<MyModal
isOpen
onClose={onClose}
iconSrc="common/running"
title={t('dataset:dataset.Training Process')}
minW={['90vw', '712px']}
isLoading={!trainingDetail && loading && tab === 'states'}
>
<ModalBody px={9} minH={['90vh', '500px']}>
<FillRowTabs
py={1}
mb={6}
value={tab}
onChange={(e) => setTab(e as 'states' | 'errors')}
list={[
{ label: t('dataset:dataset.Training Process'), value: 'states' },
{
label: t('dataset:dataset.Training_Errors', {
count: errorCounts
}),
value: 'errors'
}
]}
/>
{tab === 'states' && trainingDetail && <ProgressView trainingDetail={trainingDetail} />}
{tab === 'errors' && <ErrorView datasetId={datasetId} collectionId={collectionId} />}
</ModalBody>
</MyModal>
);
};
export default TrainingStates;

288
projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx
View File

@@ -1,110 +1,215 @@
import React from 'react';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useTranslation } from 'next-i18next';
import { Box, Button, Input, Link, ModalBody, ModalFooter } from '@chakra-ui/react';
import { strIsLink } from '@fastgpt/global/common/string/tools';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { useForm } from 'react-hook-form';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
import { getDocPath } from '@/web/common/system/doc';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { useMyStep } from '@fastgpt/web/hooks/useStep';
import MyDivider from '@fastgpt/web/components/common/MyDivider';
import React, { useRef } from 'react';
import {
Box,
Link,
Input,
Button,
ModalBody,
ModalFooter,
Textarea,
Stack
} from '@chakra-ui/react';
import {
DataChunkSplitModeEnum,
DatasetCollectionDataProcessModeEnum
} from '@fastgpt/global/core/dataset/constants';
import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import CollectionChunkForm, {
collectionChunkForm2StoreChunkData,
type CollectionChunkFormType
} from '../Form/CollectionChunkForm';
import { getLLMDefaultChunkSize } from '@fastgpt/global/core/dataset/training/utils';
import { ChunkSettingsType } from '@fastgpt/global/core/dataset/type';
type FormType = {
url?: string | undefined;
selector?: string | undefined;
export type WebsiteConfigFormType = {
websiteConfig: {
url: string;
selector: string;
};
chunkSettings: ChunkSettingsType;
};
const WebsiteConfigModal = ({
onClose,
onSuccess,
defaultValue = {
url: '',
selector: ''
}
onSuccess
}: {
onClose: () => void;
onSuccess: (data: FormType) => void;
defaultValue?: FormType;
onSuccess: (data: WebsiteConfigFormType) => void;
}) => {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const { toast } = useToast();
const { register, handleSubmit } = useForm({
defaultValues: defaultValue
const steps = [
{
title: t('dataset:website_info')
},
{
title: t('dataset:params_config')
}
];
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const websiteConfig = datasetDetail.websiteConfig;
const chunkSettings = datasetDetail.chunkSettings;
const {
register: websiteInfoForm,
handleSubmit: websiteInfoHandleSubmit,
getValues: websiteInfoGetValues
} = useForm({
defaultValues: {
url: websiteConfig?.url || '',
selector: websiteConfig?.selector || ''
}
});
const isEdit = !!defaultValue.url;
const confirmTip = isEdit
? t('common:core.dataset.website.Confirm Update Tips')
: t('common:core.dataset.website.Confirm Create Tips');
const isEdit = !!websiteConfig?.url;
const { ConfirmModal, openConfirm } = useConfirm({
type: 'common'
});
const { activeStep, goToPrevious, goToNext, MyStep } = useMyStep({
defaultStep: 0,
steps
});
const form = useForm<CollectionChunkFormType>({
defaultValues: {
trainingType: chunkSettings?.trainingType || DatasetCollectionDataProcessModeEnum.chunk,
imageIndex: chunkSettings?.imageIndex || false,
autoIndexes: chunkSettings?.autoIndexes || false,
chunkSettingMode: chunkSettings?.chunkSettingMode || ChunkSettingModeEnum.auto,
chunkSplitMode: chunkSettings?.chunkSplitMode || DataChunkSplitModeEnum.size,
embeddingChunkSize: chunkSettings?.chunkSize || 2000,
qaChunkSize: chunkSettings?.chunkSize || getLLMDefaultChunkSize(datasetDetail.agentModel),
indexSize: chunkSettings?.indexSize || datasetDetail.vectorModel?.defaultToken || 512,
chunkSplitter: chunkSettings?.chunkSplitter || '',
qaPrompt: chunkSettings?.qaPrompt || Prompt_AgentQA.description
}
});
return (
<MyModal
isOpen
iconSrc="core/dataset/websiteDataset"
title={t('common:core.dataset.website.Config')}
onClose={onClose}
maxW={'500px'}
w={'550px'}
>
<ModalBody>
<Box fontSize={'sm'} color={'myGray.600'}>
{t('common:core.dataset.website.Config Description')}
{feConfigs?.docUrl && (
<Link
href={getDocPath('/docs/guide/knowledge_base/websync/')}
target="_blank"
textDecoration={'underline'}
fontWeight={'bold'}
<ModalBody w={'full'}>
<Stack w={'75%'} marginX={'auto'}>
<MyStep />
</Stack>
<MyDivider />
{activeStep == 0 && (
<>
<Box
fontSize={'xs'}
color={'myGray.900'}
bgColor={'blue.50'}
padding={'4'}
borderRadius={'8px'}
>
{t('common:common.course.Read Course')}
</Link>
)}
</Box>
<Box mt={2}>
<Box>{t('common:core.dataset.website.Base Url')}</Box>
<Input
placeholder={t('common:core.dataset.collection.Website Link')}
{...register('url', {
required: true
})}
/>
</Box>
<Box mt={3}>
<Box>
{t('common:core.dataset.website.Selector')}({t('common:common.choosable')})
</Box>
<Input {...register('selector')} placeholder="body .content #document" />
</Box>
{t('common:core.dataset.website.Config Description')}
{feConfigs?.docUrl && (
<Link
href={getDocPath('/docs/guide/knowledge_base/websync/')}
target="_blank"
textDecoration={'underline'}
color={'blue.700'}
>
{t('common:common.course.Read Course')}
</Link>
)}
</Box>
<Box mt={2}>
<Box>{t('common:core.dataset.website.Base Url')}</Box>
<Input
placeholder={t('common:core.dataset.collection.Website Link')}
{...websiteInfoForm('url', {
required: true
})}
/>
</Box>
<Box mt={3}>
<Box>
{t('common:core.dataset.website.Selector')}({t('common:common.choosable')})
</Box>
<Input {...websiteInfoForm('selector')} placeholder="body .content #document" />
</Box>
</>
)}
{activeStep == 1 && <CollectionChunkForm form={form} />}
</ModalBody>
<ModalFooter>
<Button variant={'whiteBase'} onClick={onClose}>
{t('common:common.Close')}
</Button>
<Button
ml={2}
onClick={handleSubmit((data) => {
if (!data.url) return;
// check is link
if (!strIsLink(data.url)) {
return toast({
status: 'warning',
title: t('common:common.link.UnValid')
});
}
openConfirm(
() => {
onSuccess(data);
},
undefined,
confirmTip
)();
})}
>
{t('common:core.dataset.website.Start Sync')}
</Button>
{activeStep == 0 && (
<>
<Button variant={'whiteBase'} onClick={onClose}>
{t('common:common.Close')}
</Button>
<Button
ml={2}
onClick={websiteInfoHandleSubmit((data) => {
if (!data.url) return;
// check is link
if (!strIsLink(data.url)) {
return toast({
status: 'warning',
title: t('common:common.link.UnValid')
});
}
goToNext();
})}
>
{t('common:common.Next Step')}
</Button>
</>
)}
{activeStep == 1 && (
<>
<Button variant={'whiteBase'} onClick={goToPrevious}>
{t('common:common.Last Step')}
</Button>
<Button
ml={2}
onClick={form.handleSubmit((data) => {
openConfirm(
() =>
onSuccess({
websiteConfig: websiteInfoGetValues(),
chunkSettings: collectionChunkForm2StoreChunkData({
...data,
agentModel: datasetDetail.agentModel,
vectorModel: datasetDetail.vectorModel
})
}),
undefined,
isEdit
? t('common:core.dataset.website.Confirm Update Tips')
: t('common:core.dataset.website.Confirm Create Tips')
)();
})}
>
{t('common:core.dataset.website.Start Sync')}
</Button>
</>
)}
</ModalFooter>
<ConfirmModal />
</MyModal>
@@ -112,3 +217,42 @@ const WebsiteConfigModal = ({
};
export default WebsiteConfigModal;
const PromptTextarea = ({
defaultValue,
onChange,
onClose
}: {
defaultValue: string;
onChange: (e: string) => void;
onClose: () => void;
}) => {
const ref = useRef<HTMLTextAreaElement>(null);
const { t } = useTranslation();
return (
<MyModal
isOpen
title={t('common:core.dataset.import.Custom prompt')}
iconSrc="modal/edit"
w={'600px'}
onClose={onClose}
>
<ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
<Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
<Box>{Prompt_AgentQA.fixedText}</Box>
</ModalBody>
<ModalFooter>
<Button
onClick={() => {
const val = ref.current?.value || Prompt_AgentQA.description;
onChange(val);
onClose();
}}
>
{t('common:common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};

73
projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx
View File

@@ -51,6 +51,7 @@ import {
import { useFolderDrag } from '@/components/common/folder/useFolderDrag';
import TagsPopOver from './TagsPopOver';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import TrainingStates from './TrainingStates';
const Header = dynamic(() => import('./Header'));
const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip'));
@@ -63,26 +64,25 @@ const CollectionCard = () => {
const { datasetDetail, loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v);
const { feConfigs } = useSystemStore();
const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
content: t('common:dataset.Confirm to delete the file'),
type: 'delete'
});
const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('common:Rename')
});
const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>();
const [trainingStatesCollection, setTrainingStatesCollection] = useState<{
collectionId: string;
}>();
const { collections, Pagination, total, getData, isGetting, pageNum, pageSize } =
useContextSelector(CollectionPageContext, (v) => v);
// Ad file status icon
// Add file status icon
const formatCollections = useMemo(
() =>
collections.map((collection) => {
const icon = getCollectionIcon(collection.type, collection.name);
const status = (() => {
if (collection.hasError) {
return {
statusText: t('common:core.dataset.collection.status.error'),
colorSchema: 'red'
};
}
if (collection.trainingAmount > 0) {
return {
statusText: t('common:dataset.collections.Collection Embedding', {
@@ -106,6 +106,11 @@ const CollectionCard = () => {
[collections, t]
);
const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>();
const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
title: t('common:Rename')
});
const { runAsync: onUpdateCollection, loading: isUpdating } = useRequest2(
putDatasetCollectionById,
{
@@ -115,7 +120,12 @@ const CollectionCard = () => {
successToast: t('common:common.Update Success')
}
);
const { runAsync: onDelCollection, loading: isDeleting } = useRequest2(
const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
content: t('common:dataset.Confirm to delete the file'),
type: 'delete'
});
const { runAsync: onDelCollection } = useRequest2(
(collectionId: string) => {
return delDatasetCollectionById({
id: collectionId
@@ -153,14 +163,14 @@ const CollectionCard = () => {
['refreshCollection'],
() => {
getData(pageNum);
if (datasetDetail.status === DatasetStatusEnum.syncing) {
if (datasetDetail.status !== DatasetStatusEnum.active) {
loadDatasetDetail(datasetDetail._id);
}
return null;
},
{
refetchInterval: 6000,
enabled: hasTrainingData || datasetDetail.status === DatasetStatusEnum.syncing
enabled: hasTrainingData || datasetDetail.status !== DatasetStatusEnum.active
}
);
@@ -180,13 +190,13 @@ const CollectionCard = () => {
});
const isLoading =
isUpdating || isDeleting || isSyncing || (isGetting && collections.length === 0) || isDropping;
isUpdating || isSyncing || (isGetting && collections.length === 0) || isDropping;
return (
<MyBox isLoading={isLoading} h={'100%'} py={[2, 4]}>
<Flex ref={BoxRef} flexDirection={'column'} py={[1, 0]} h={'100%'} px={[2, 6]}>
{/* header */}
<Header />
<Header hasTrainingData={hasTrainingData} />
{/* collection table */}
<TableContainer mt={3} overflowY={'auto'} fontSize={'sm'}>
@@ -269,9 +279,22 @@ const CollectionCard = () => {
<Box>{formatTime2YMDHM(collection.updateTime)}</Box>
</Td>
<Td py={2}>
<MyTag showDot colorSchema={collection.colorSchema as any} type={'borderFill'}>
{t(collection.statusText as any)}
</MyTag>
<MyTooltip label={t('common:Click_to_expand')}>
<MyTag
showDot
colorSchema={collection.colorSchema as any}
type={'fill'}
onClick={(e) => {
e.stopPropagation();
setTrainingStatesCollection({ collectionId: collection._id });
}}
>
<Flex fontWeight={'medium'} alignItems={'center'} gap={1}>
{t(collection.statusText as any)}
<MyIcon name={'common/maximize'} w={'11px'} />
</Flex>
</MyTag>
</MyTooltip>
</Td>
<Td py={2} onClick={(e) => e.stopPropagation()}>
<Switch
@@ -383,9 +406,7 @@ const CollectionCard = () => {
type: 'danger',
onClick: () =>
openDeleteConfirm(
() => {
onDelCollection(collection._id);
},
() => onDelCollection(collection._id),
undefined,
collection.type === DatasetCollectionTypeEnum.folder
? t('common:dataset.collections.Confirm to delete the folder')
@@ -414,6 +435,14 @@ const CollectionCard = () => {
<ConfirmSyncModal />
<EditTitleModal />
{!!trainingStatesCollection && (
<TrainingStates
datasetId={datasetDetail._id}
collectionId={trainingStatesCollection.collectionId}
onClose={() => setTrainingStatesCollection(undefined)}
/>
)}
{!!moveCollectionData && (
<SelectCollections
datasetId={datasetDetail._id}

31
projects/app/src/pageComponents/dataset/detail/DataCard.tsx
View File

@@ -30,6 +30,7 @@ import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
import { TabEnum } from './NavBar';
import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import TrainingStates from './CollectionCard/TrainingStates';
const DataCard = () => {
const theme = useTheme();
@@ -44,6 +45,7 @@ const DataCard = () => {
const { t } = useTranslation();
const [searchText, setSearchText] = useState('');
const [errorModalId, setErrorModalId] = useState('');
const { toast } = useToast();
const scrollParams = useMemo(
@@ -174,7 +176,7 @@ const DataCard = () => {
<MyDivider my={'17px'} w={'100%'} />
</Box>
<Flex alignItems={'center'} px={6} pb={4}>
<Flex align={'center'} color={'myGray.500'}>
<Flex alignItems={'center'} color={'myGray.500'}>
<MyIcon name="common/list" mr={2} w={'18px'} />
<Box as={'span'} fontSize={['sm', '14px']} fontWeight={'500'}>
{t('dataset:data_amount', {
@@ -182,6 +184,25 @@ const DataCard = () => {
indexAmount: collection?.indexAmount ?? '-'
})}
</Box>
{!!collection?.errorCount && (
<MyTag
colorSchema={'red'}
type={'fill'}
cursor={'pointer'}
rounded={'full'}
ml={2}
onClick={() => {
setErrorModalId(collection._id);
}}
>
<Flex fontWeight={'medium'} alignItems={'center'} gap={1}>
{t('dataset:data_error_amount', {
errorAmount: collection?.errorCount
})}
<MyIcon name={'common/maximize'} w={'11px'} />
</Flex>
</MyTag>
)}
</Flex>
<Box flex={1} mr={1} />
<MyInput
@@ -354,6 +375,14 @@ const DataCard = () => {
}}
/>
)}
{errorModalId && (
<TrainingStates
datasetId={datasetId}
defaultTab={'errors'}
collectionId={errorModalId}
onClose={() => setErrorModalId('')}
/>
)}
<ConfirmModal />
</MyBox>
);

524
projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx Normal file
View File

@@ -0,0 +1,524 @@
import MyModal from '@fastgpt/web/components/common/MyModal';
import { useTranslation } from 'next-i18next';
import { UseFormReturn } from 'react-hook-form';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import React, { useEffect, useMemo, useRef, useState } from 'react';
import {
Box,
Flex,
Input,
Button,
ModalBody,
ModalFooter,
Textarea,
useDisclosure,
Checkbox,
HStack
} from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import {
DataChunkSplitModeEnum,
DatasetCollectionDataProcessModeEnum,
DatasetCollectionDataProcessModeMap
} from '@fastgpt/global/core/dataset/constants';
import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
import { useContextSelector } from 'use-context-selector';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import MySelect from '@fastgpt/web/components/common/MySelect';
import {
chunkAutoChunkSize,
getAutoIndexSize,
getIndexSizeSelectList,
getLLMDefaultChunkSize,
getLLMMaxChunkSize,
getMaxChunkSize,
getMaxIndexSize,
minChunkSize
} from '@fastgpt/global/core/dataset/training/utils';
import RadioGroup from '@fastgpt/web/components/common/Radio/RadioGroup';
import { ChunkSettingsType } from '@fastgpt/global/core/dataset/type';
import type { LLMModelItemType, EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.d';
const PromptTextarea = ({
defaultValue = '',
onChange,
onClose
}: {
defaultValue?: string;
onChange: (e: string) => void;
onClose: () => void;
}) => {
const ref = useRef<HTMLTextAreaElement>(null);
const { t } = useTranslation();
return (
<MyModal
isOpen
title={t('common:core.dataset.import.Custom prompt')}
iconSrc="modal/edit"
w={'600px'}
onClose={onClose}
>
<ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
<Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
<Box>{Prompt_AgentQA.fixedText}</Box>
</ModalBody>
<ModalFooter>
<Button
onClick={() => {
const val = ref.current?.value || Prompt_AgentQA.description;
onChange(val);
onClose();
}}
>
{t('common:common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};
export type CollectionChunkFormType = {
trainingType: DatasetCollectionDataProcessModeEnum;
imageIndex: boolean;
autoIndexes: boolean;
chunkSettingMode: ChunkSettingModeEnum;
chunkSplitMode: DataChunkSplitModeEnum;
embeddingChunkSize: number;
qaChunkSize: number;
chunkSplitter?: string;
indexSize: number;
qaPrompt?: string;
};
const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkFormType> }) => {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const vectorModel = datasetDetail.vectorModel;
const agentModel = datasetDetail.agentModel;
const { setValue, register, watch, getValues } = form;
const trainingType = watch('trainingType');
const chunkSettingMode = watch('chunkSettingMode');
const chunkSplitMode = watch('chunkSplitMode');
const autoIndexes = watch('autoIndexes');
const indexSize = watch('indexSize');
const trainingModeList = useMemo(() => {
const list = Object.entries(DatasetCollectionDataProcessModeMap);
return list
.filter(([key]) => key !== DatasetCollectionDataProcessModeEnum.auto)
.map(([key, value]) => ({
title: t(value.label as any),
value: key as DatasetCollectionDataProcessModeEnum,
tooltip: t(value.tooltip as any)
}));
}, [t]);
const {
chunkSizeField,
maxChunkSize,
minChunkSize: minChunkSizeValue,
maxIndexSize
} = useMemo(() => {
if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
return {
chunkSizeField: 'qaChunkSize',
maxChunkSize: getLLMMaxChunkSize(agentModel),
minChunkSize: 1000,
maxIndexSize: 1000
};
} else if (autoIndexes) {
return {
chunkSizeField: 'embeddingChunkSize',
maxChunkSize: getMaxChunkSize(agentModel),
minChunkSize: minChunkSize,
maxIndexSize: getMaxIndexSize(vectorModel)
};
} else {
return {
chunkSizeField: 'embeddingChunkSize',
maxChunkSize: getMaxChunkSize(agentModel),
minChunkSize: minChunkSize,
maxIndexSize: getMaxIndexSize(vectorModel)
};
}
}, [trainingType, autoIndexes, agentModel, vectorModel]);
// Custom split list
const customSplitList = [
{ label: t('dataset:split_sign_null'), value: '' },
{ label: t('dataset:split_sign_break'), value: '\\n' },
{ label: t('dataset:split_sign_break2'), value: '\\n\\n' },
{ label: t('dataset:split_sign_period'), value: '.|。' },
{ label: t('dataset:split_sign_exclamatiob'), value: '!|！' },
{ label: t('dataset:split_sign_question'), value: '?|？' },
{ label: t('dataset:split_sign_semicolon'), value: ';|；' },
{ label: '=====', value: '=====' },
{ label: t('dataset:split_sign_custom'), value: 'Other' }
];
const [customListSelectValue, setCustomListSelectValue] = useState(getValues('chunkSplitter'));
useEffect(() => {
if (customListSelectValue === 'Other') {
setValue('chunkSplitter', '');
} else {
setValue('chunkSplitter', customListSelectValue);
}
}, [customListSelectValue, setValue]);
// Index size
const indexSizeSeletorList = useMemo(() => getIndexSizeSelectList(maxIndexSize), [maxIndexSize]);
// QA
const qaPrompt = watch('qaPrompt');
const {
isOpen: isOpenCustomPrompt,
onOpen: onOpenCustomPrompt,
onClose: onCloseCustomPrompt
} = useDisclosure();
const showQAPromptInput = trainingType === DatasetCollectionDataProcessModeEnum.qa;
// Adapt 4.9.0- auto training
useEffect(() => {
if (trainingType === DatasetCollectionDataProcessModeEnum.auto) {
setValue('autoIndexes', true);
setValue('trainingType', DatasetCollectionDataProcessModeEnum.chunk);
}
}, [trainingType, setValue]);
return (
<>
<Box>
<Box fontSize={'sm'} mb={2} color={'myGray.600'}>
{t('dataset:training_mode')}
</Box>
<LeftRadio<DatasetCollectionDataProcessModeEnum>
list={trainingModeList}
px={3}
py={2.5}
value={trainingType}
onChange={(e) => {
setValue('trainingType', e);
}}
defaultBg="white"
activeBg="white"
gridTemplateColumns={'repeat(2, 1fr)'}
/>
</Box>
{trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
<Box mt={6}>
<Box fontSize={'sm'} mb={2} color={'myGray.600'}>
{t('dataset:enhanced_indexes')}
</Box>
<HStack gap={[3, 7]}>
<HStack flex={'1'} spacing={1}>
<MyTooltip label={!feConfigs?.isPlus ? t('common:commercial_function_tip') : ''}>
<Checkbox isDisabled={!feConfigs?.isPlus} {...register('autoIndexes')}>
<FormLabel>{t('dataset:auto_indexes')}</FormLabel>
</Checkbox>
</MyTooltip>
<QuestionTip label={t('dataset:auto_indexes_tips')} />
</HStack>
<HStack flex={'1'} spacing={1}>
<MyTooltip
label={
!feConfigs?.isPlus
? t('common:commercial_function_tip')
: !datasetDetail?.vlmModel
? t('common:error_vlm_not_config')
: ''
}
>
<Checkbox
isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
{...register('imageIndex')}
>
<FormLabel>{t('dataset:image_auto_parse')}</FormLabel>
</Checkbox>
</MyTooltip>
<QuestionTip label={t('dataset:image_auto_parse_tips')} />
</HStack>
</HStack>
</Box>
)}
<Box mt={6}>
<Box fontSize={'sm'} mb={2} color={'myGray.600'}>
{t('dataset:params_setting')}
</Box>
<LeftRadio<ChunkSettingModeEnum>
list={[
{
title: t('dataset:default_params'),
desc: t('dataset:default_params_desc'),
value: ChunkSettingModeEnum.auto
},
{
title: t('dataset:custom_data_process_params'),
desc: t('dataset:custom_data_process_params_desc'),
value: ChunkSettingModeEnum.custom,
children: chunkSettingMode === ChunkSettingModeEnum.custom && (
<Box mt={5}>
<Box>
<RadioGroup<DataChunkSplitModeEnum>
list={[
{
title: t('dataset:split_chunk_size'),
value: DataChunkSplitModeEnum.size
},
{
title: t('dataset:split_chunk_char'),
value: DataChunkSplitModeEnum.char,
tooltip: t('dataset:custom_split_sign_tip')
}
]}
value={chunkSplitMode}
onChange={(e) => {
setValue('chunkSplitMode', e);
}}
/>
{chunkSplitMode === DataChunkSplitModeEnum.size && (
<Box
mt={1.5}
css={{
'& > span': {
display: 'block'
}
}}
>
<MyTooltip
label={t('common:core.dataset.import.Chunk Range', {
min: minChunkSizeValue,
max: maxChunkSize
})}
>
<MyNumberInput
register={register}
name={chunkSizeField}
min={minChunkSizeValue}
max={maxChunkSize}
size={'sm'}
step={100}
/>
</MyTooltip>
</Box>
)}
{chunkSplitMode === DataChunkSplitModeEnum.char && (
<HStack mt={1.5}>
<Box flex={'1 0 0'}>
<MySelect<string>
list={customSplitList}
size={'sm'}
bg={'myGray.50'}
value={customListSelectValue}
h={'32px'}
onChange={(val) => {
setCustomListSelectValue(val);
}}
/>
</Box>
{customListSelectValue === 'Other' && (
<Input
flex={'1 0 0'}
h={'32px'}
size={'sm'}
bg={'myGray.50'}
placeholder="\n;======;==SPLIT=="
{...register('chunkSplitter')}
/>
)}
</HStack>
)}
</Box>
{trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
<Box>
<Flex alignItems={'center'} mt={3}>
<Box>{t('dataset:index_size')}</Box>
<QuestionTip label={t('dataset:index_size_tips')} />
</Flex>
<Box mt={1}>
<MySelect<number>
bg={'myGray.50'}
list={indexSizeSeletorList}
value={indexSize}
onChange={(val) => {
setValue('indexSize', val);
}}
/>
</Box>
</Box>
)}
{showQAPromptInput && (
<Box mt={3}>
<Box>{t('common:core.dataset.collection.QA Prompt')}</Box>
<Box
position={'relative'}
py={2}
px={3}
bg={'myGray.50'}
fontSize={'xs'}
whiteSpace={'pre-wrap'}
border={'1px'}
borderColor={'borderColor.base'}
borderRadius={'md'}
maxH={'140px'}
overflow={'auto'}
_hover={{
'& .mask': {
display: 'block'
}
}}
>
{qaPrompt}
<Box
display={'none'}
className="mask"
position={'absolute'}
top={0}
right={0}
bottom={0}
left={0}
background={
'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
}
>
<Button
size="xs"
variant={'whiteBase'}
leftIcon={<MyIcon name={'edit'} w={'13px'} />}
color={'black'}
position={'absolute'}
right={2}
bottom={2}
onClick={onOpenCustomPrompt}
>
{t('common:core.dataset.import.Custom prompt')}
</Button>
</Box>
</Box>
</Box>
)}
</Box>
)
}
]}
gridGap={3}
px={3}
py={3}
defaultBg="white"
activeBg="white"
value={chunkSettingMode}
w={'100%'}
onChange={(e) => {
setValue('chunkSettingMode', e);
}}
/>
</Box>
{isOpenCustomPrompt && (
<PromptTextarea
defaultValue={qaPrompt}
onChange={(e) => {
setValue('qaPrompt', e);
}}
onClose={onCloseCustomPrompt}
/>
)}
</>
);
};
export default CollectionChunkForm;
export const collectionChunkForm2StoreChunkData = ({
trainingType,
imageIndex,
autoIndexes,
chunkSettingMode,
chunkSplitMode,
embeddingChunkSize,
qaChunkSize,
chunkSplitter,
indexSize,
qaPrompt,
agentModel,
vectorModel
}: CollectionChunkFormType & {
agentModel: LLMModelItemType;
vectorModel: EmbeddingModelItemType;
}): ChunkSettingsType => {
const trainingModeSize: {
autoChunkSize: number;
autoIndexSize: number;
chunkSize: number;
indexSize: number;
} = (() => {
if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
return {
autoChunkSize: getLLMDefaultChunkSize(agentModel),
autoIndexSize: 512,
chunkSize: qaChunkSize,
indexSize: 512
};
} else if (autoIndexes) {
return {
autoChunkSize: chunkAutoChunkSize,
autoIndexSize: getAutoIndexSize(vectorModel),
chunkSize: embeddingChunkSize,
indexSize
};
} else {
return {
autoChunkSize: chunkAutoChunkSize,
autoIndexSize: getAutoIndexSize(vectorModel),
chunkSize: embeddingChunkSize,
indexSize
};
}
})();
const { chunkSize: formatChunkIndex, indexSize: formatIndexSize } = (() => {
if (chunkSettingMode === ChunkSettingModeEnum.auto) {
return {
chunkSize: trainingModeSize.autoChunkSize,
indexSize: trainingModeSize.autoIndexSize
};
} else {
return {
chunkSize: trainingModeSize.chunkSize,
indexSize: trainingModeSize.indexSize
};
}
})();
return {
trainingType,
imageIndex,
autoIndexes,
chunkSettingMode,
chunkSplitMode,
chunkSize: formatChunkIndex,
indexSize: formatIndexSize,
chunkSplitter,
qaPrompt: trainingType === DatasetCollectionDataProcessModeEnum.qa ? qaPrompt : undefined
};
};

37
projects/app/src/pageComponents/dataset/detail/Import/Context.tsx
View File

@@ -25,6 +25,14 @@ import {
getAutoIndexSize,
getMaxIndexSize
} from '@fastgpt/global/core/dataset/training/utils';
import { CollectionChunkFormType } from '../Form/CollectionChunkForm';
type ChunkSizeFieldType = 'embeddingChunkSize' | 'qaChunkSize';
export type ImportFormType = {
customPdfParse: boolean;
webSelector: string;
} & CollectionChunkFormType;
type TrainingFiledType = {
chunkOverlapRatio: number;
@@ -51,26 +59,6 @@ type DatasetImportContextType = {
setSources: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
} & TrainingFiledType;
type ChunkSizeFieldType = 'embeddingChunkSize' | 'qaChunkSize';
export type ImportFormType = {
customPdfParse: boolean;
trainingType: DatasetCollectionDataProcessModeEnum;
imageIndex: boolean;
autoIndexes: boolean;
chunkSettingMode: ChunkSettingModeEnum;
chunkSplitMode: DataChunkSplitModeEnum;
embeddingChunkSize: number;
qaChunkSize: number;
chunkSplitter: string;
indexSize: number;
qaPrompt: string;
webSelector: string;
};
export const DatasetImportContext = createContext<DatasetImportContextType>({
importSource: ImportDataSourceEnum.fileLocal,
goToNext: function (): void {
@@ -314,14 +302,7 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
chunkSplitter
};
}
}, [
chunkSettingMode,
TrainingModeMap.autoChunkSize,
TrainingModeMap.autoIndexSize,
TrainingModeMap.chunkSize,
TrainingModeMap.indexSize,
chunkSplitter
]);
}, [chunkSettingMode, TrainingModeMap, chunkSplitter]);
const contextValue = {
...TrainingModeMap,

380
projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx
View File

@@ -1,13 +1,8 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import React, { useCallback } from 'react';
import {
Box,
Flex,
Input,
Button,
ModalBody,
ModalFooter,
Textarea,
useDisclosure,
Checkbox,
Accordion,
AccordionItem,
@@ -16,93 +11,26 @@ import {
AccordionIcon,
HStack
} from '@chakra-ui/react';
import MyIcon from '@fastgpt/web/components/common/Icon';
import { useTranslation } from 'next-i18next';
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import {
DataChunkSplitModeEnum,
DatasetCollectionDataProcessModeEnum,
DatasetCollectionDataProcessModeMap
} from '@fastgpt/global/core/dataset/constants';
import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
import MyTag from '@fastgpt/web/components/common/Tag/index';
import { useContextSelector } from 'use-context-selector';
import { DatasetImportContext } from '../Context';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import { shadowLight } from '@fastgpt/web/styles/theme';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import MySelect from '@fastgpt/web/components/common/MySelect';
import { getIndexSizeSelectList } from '@fastgpt/global/core/dataset/training/utils';
import RadioGroup from '@fastgpt/web/components/common/Radio/RadioGroup';
import CollectionChunkForm from '../../Form/CollectionChunkForm';
import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
function DataProcess() {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const {
goToNext,
processParamsForm,
chunkSizeField,
minChunkSize,
maxChunkSize,
maxIndexSize,
indexSize
} = useContextSelector(DatasetImportContext, (v) => v);
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { setValue, register, watch, getValues } = processParamsForm;
const trainingType = watch('trainingType');
const trainingModeList = useMemo(() => {
const list = Object.entries(DatasetCollectionDataProcessModeMap);
return list
.filter(([key]) => key !== DatasetCollectionDataProcessModeEnum.auto)
.map(([key, value]) => ({
title: t(value.label as any),
value: key as DatasetCollectionDataProcessModeEnum,
tooltip: t(value.tooltip as any)
}));
}, [t]);
const chunkSettingMode = watch('chunkSettingMode');
const chunkSplitMode = watch('chunkSplitMode');
const customSplitList = [
{ label: t('dataset:split_sign_null'), value: '' },
{ label: t('dataset:split_sign_break'), value: '\\n' },
{ label: t('dataset:split_sign_break2'), value: '\\n\\n' },
{ label: t('dataset:split_sign_period'), value: '.|。' },
{ label: t('dataset:split_sign_exclamatiob'), value: '!|！' },
{ label: t('dataset:split_sign_question'), value: '?|？' },
{ label: t('dataset:split_sign_semicolon'), value: ';|；' },
{ label: '=====', value: '=====' },
{ label: t('dataset:split_sign_custom'), value: 'Other' }
];
const [customListSelectValue, setCustomListSelectValue] = useState(getValues('chunkSplitter'));
useEffect(() => {
if (customListSelectValue === 'Other') {
setValue('chunkSplitter', '');
} else {
setValue('chunkSplitter', customListSelectValue);
}
}, [customListSelectValue, setValue]);
// Index size
const indexSizeSeletorList = useMemo(() => getIndexSizeSelectList(maxIndexSize), [maxIndexSize]);
// QA
const qaPrompt = watch('qaPrompt');
const {
isOpen: isOpenCustomPrompt,
onOpen: onOpenCustomPrompt,
onClose: onCloseCustomPrompt
} = useDisclosure();
const { goToNext, processParamsForm, chunkSize } = useContextSelector(
DatasetImportContext,
(v) => v
);
const { register } = processParamsForm;
const Title = useCallback(({ title }: { title: string }) => {
return (
@@ -116,16 +44,7 @@ function DataProcess() {
);
}, []);
// Adapt auto training
useEffect(() => {
if (trainingType === DatasetCollectionDataProcessModeEnum.auto) {
setValue('autoIndexes', true);
setValue('trainingType', DatasetCollectionDataProcessModeEnum.chunk);
}
}, [trainingType, setValue]);
const showFileParseSetting = feConfigs?.showCustomPdfParse;
const showQAPromptInput = trainingType === DatasetCollectionDataProcessModeEnum.qa;
return (
<>
@@ -179,238 +98,8 @@ function DataProcess() {
<Title title={t('dataset:import_data_process_setting')} />
<AccordionPanel p={2}>
<Box mt={2}>
<Box fontSize={'sm'} mb={2} color={'myGray.600'}>
{t('dataset:training_mode')}
</Box>
<LeftRadio<DatasetCollectionDataProcessModeEnum>
list={trainingModeList}
px={3}
py={2.5}
value={trainingType}
onChange={(e) => {
setValue('trainingType', e);
}}
defaultBg="white"
activeBg="white"
gridTemplateColumns={'repeat(2, 1fr)'}
/>
</Box>
{trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
<Box mt={6}>
<Box fontSize={'sm'} mb={2} color={'myGray.600'}>
{t('dataset:enhanced_indexes')}
</Box>
<HStack gap={[3, 7]}>
<HStack flex={'1'} spacing={1}>
<MyTooltip
label={!feConfigs?.isPlus ? t('common:commercial_function_tip') : ''}
>
<Checkbox isDisabled={!feConfigs?.isPlus} {...register('autoIndexes')}>
<FormLabel>{t('dataset:auto_indexes')}</FormLabel>
</Checkbox>
</MyTooltip>
<QuestionTip label={t('dataset:auto_indexes_tips')} />
</HStack>
<HStack flex={'1'} spacing={1}>
<MyTooltip
label={
!feConfigs?.isPlus
? t('common:commercial_function_tip')
: !datasetDetail?.vlmModel
? t('common:error_vlm_not_config')
: ''
}
>
<Checkbox
isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
{...register('imageIndex')}
>
<FormLabel>{t('dataset:image_auto_parse')}</FormLabel>
</Checkbox>
</MyTooltip>
<QuestionTip label={t('dataset:image_auto_parse_tips')} />
</HStack>
</HStack>
</Box>
)}
<Box mt={6}>
<Box fontSize={'sm'} mb={2} color={'myGray.600'}>
{t('dataset:params_setting')}
</Box>
<LeftRadio<ChunkSettingModeEnum>
list={[
{
title: t('dataset:default_params'),
desc: t('dataset:default_params_desc'),
value: ChunkSettingModeEnum.auto
},
{
title: t('dataset:custom_data_process_params'),
desc: t('dataset:custom_data_process_params_desc'),
value: ChunkSettingModeEnum.custom,
children: chunkSettingMode === ChunkSettingModeEnum.custom && (
<Box mt={5}>
<Box>
<RadioGroup<DataChunkSplitModeEnum>
list={[
{
title: t('dataset:split_chunk_size'),
value: DataChunkSplitModeEnum.size
},
{
title: t('dataset:split_chunk_char'),
value: DataChunkSplitModeEnum.char,
tooltip: t('dataset:custom_split_sign_tip')
}
]}
value={chunkSplitMode}
onChange={(e) => {
setValue('chunkSplitMode', e);
}}
/>
{chunkSplitMode === DataChunkSplitModeEnum.size && (
<Box
mt={1.5}
css={{
'& > span': {
display: 'block'
}
}}
>
<MyTooltip
label={t('common:core.dataset.import.Chunk Range', {
min: minChunkSize,
max: maxChunkSize
})}
>
<MyNumberInput
register={register}
name={chunkSizeField}
min={minChunkSize}
max={maxChunkSize}
size={'sm'}
step={100}
/>
</MyTooltip>
</Box>
)}
{chunkSplitMode === DataChunkSplitModeEnum.char && (
<HStack mt={1.5}>
<Box flex={'1 0 0'}>
<MySelect<string>
list={customSplitList}
size={'sm'}
bg={'myGray.50'}
value={customListSelectValue}
h={'32px'}
onChange={(val) => {
setCustomListSelectValue(val);
}}
/>
</Box>
{customListSelectValue === 'Other' && (
<Input
flex={'1 0 0'}
h={'32px'}
size={'sm'}
bg={'myGray.50'}
placeholder="\n;======;==SPLIT=="
{...register('chunkSplitter')}
/>
)}
</HStack>
)}
</Box>
{trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
<Box>
<Flex alignItems={'center'} mt={3}>
<Box>{t('dataset:index_size')}</Box>
<QuestionTip label={t('dataset:index_size_tips')} />
</Flex>
<Box mt={1}>
<MySelect<number>
bg={'myGray.50'}
list={indexSizeSeletorList}
value={indexSize}
onChange={(val) => {
setValue('indexSize', val);
}}
/>
</Box>
</Box>
)}
{showQAPromptInput && (
<Box mt={3}>
<Box>{t('common:core.dataset.collection.QA Prompt')}</Box>
<Box
position={'relative'}
py={2}
px={3}
bg={'myGray.50'}
fontSize={'xs'}
whiteSpace={'pre-wrap'}
border={'1px'}
borderColor={'borderColor.base'}
borderRadius={'md'}
maxH={'140px'}
overflow={'auto'}
_hover={{
'& .mask': {
display: 'block'
}
}}
>
{qaPrompt}
<Box
display={'none'}
className="mask"
position={'absolute'}
top={0}
right={0}
bottom={0}
left={0}
background={
'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
}
>
<Button
size="xs"
variant={'whiteBase'}
leftIcon={<MyIcon name={'edit'} w={'13px'} />}
color={'black'}
position={'absolute'}
right={2}
bottom={2}
onClick={onOpenCustomPrompt}
>
{t('common:core.dataset.import.Custom prompt')}
</Button>
</Box>
</Box>
</Box>
)}
</Box>
)
}
]}
gridGap={3}
px={3}
py={3}
defaultBg="white"
activeBg="white"
value={chunkSettingMode}
w={'100%'}
onChange={(e) => {
setValue('chunkSettingMode', e);
}}
/>
</Box>
{/* @ts-ignore */}
<CollectionChunkForm form={processParamsForm} />
</AccordionPanel>
</AccordionItem>
@@ -425,57 +114,8 @@ function DataProcess() {
</Flex>
</Accordion>
</Box>
{isOpenCustomPrompt && (
<PromptTextarea
defaultValue={qaPrompt}
onChange={(e) => {
setValue('qaPrompt', e);
}}
onClose={onCloseCustomPrompt}
/>
)}
</>
);
}
export default React.memo(DataProcess);
const PromptTextarea = ({
defaultValue,
onChange,
onClose
}: {
defaultValue: string;
onChange: (e: string) => void;
onClose: () => void;
}) => {
const ref = useRef<HTMLTextAreaElement>(null);
const { t } = useTranslation();
return (
<MyModal
isOpen
title={t('common:core.dataset.import.Custom prompt')}
iconSrc="modal/edit"
w={'600px'}
onClose={onClose}
>
<ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
<Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
<Box>{Prompt_AgentQA.fixedText}</Box>
</ModalBody>
<ModalFooter>
<Button
onClick={() => {
const val = ref.current?.value || Prompt_AgentQA.description;
onChange(val);
onClose();
}}
>
{t('common:common.Confirm')}
</Button>
</ModalFooter>
</MyModal>
);
};

6
projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx
View File

@@ -85,9 +85,13 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
value: t(DatasetCollectionDataProcessModeMap[collection.trainingType]?.label as any)
},
{
label: t('common:core.dataset.collection.metadata.Chunk Size'),
label: t('dataset:chunk_size'),
value: collection.chunkSize || '-'
},
{
label: t('dataset:index_size'),
value: collection.indexSize || '-'
},
...(webSelector
? [
{