gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: dataset import params code (#4875)

Browse Source 
* perf: dataset import params code

* perf: api dataset code

* model
This commit is contained in:
Archer
2025-05-23 10:40:25 +08:00
committed by GitHub
parent 9af92d1eae
commit fae76e887a
23 changed files with 366 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx
View File

@@ -21,9 +21,13 @@ import CollectionChunkForm, {
collectionChunkForm2StoreChunkData,
type CollectionChunkFormType
} from '../Form/CollectionChunkForm';
import { getLLMDefaultChunkSize } from '@fastgpt/global/core/dataset/training/utils';
import {
getAutoIndexSize,
getLLMDefaultChunkSize
} from '@fastgpt/global/core/dataset/training/utils';
import { type ChunkSettingsType } from '@fastgpt/global/core/dataset/type';
import PopoverConfirm from '@fastgpt/web/components/common/MyPopover/PopoverConfirm';
import { defaultFormData } from '../Import/Context';
export type WebsiteConfigFormType = {
websiteConfig: {
@@ -76,17 +80,35 @@ const WebsiteConfigModal = ({
const form = useForm<CollectionChunkFormType>({
defaultValues: {
trainingType: chunkSettings?.trainingType || DatasetCollectionDataProcessModeEnum.chunk,
imageIndex: chunkSettings?.imageIndex || false,
autoIndexes: chunkSettings?.autoIndexes || false,
trainingType: chunkSettings?.trainingType,
chunkSettingMode: chunkSettings?.chunkSettingMode || ChunkSettingModeEnum.auto,
chunkSplitMode: chunkSettings?.chunkSplitMode || DataChunkSplitModeEnum.size,
embeddingChunkSize: chunkSettings?.chunkSize || 2000,
qaChunkSize: chunkSettings?.chunkSize || getLLMDefaultChunkSize(datasetDetail.agentModel),
indexSize: chunkSettings?.indexSize || datasetDetail.vectorModel?.defaultToken || 512,
chunkTriggerType: chunkSettings?.chunkTriggerType || defaultFormData.chunkTriggerType,
chunkTriggerMinSize:
chunkSettings?.chunkTriggerMinSize || defaultFormData.chunkTriggerMinSize,
dataEnhanceCollectionName:
chunkSettings?.dataEnhanceCollectionName || defaultFormData.dataEnhanceCollectionName,
imageIndex: chunkSettings?.imageIndex || defaultFormData.imageIndex,
autoIndexes: chunkSettings?.autoIndexes || defaultFormData.autoIndexes,
chunkSettingMode: chunkSettings?.chunkSettingMode || defaultFormData.chunkSettingMode,
chunkSplitMode: chunkSettings?.chunkSplitMode || defaultFormData.chunkSplitMode,
paragraphChunkAIMode:
chunkSettings?.paragraphChunkAIMode || defaultFormData.paragraphChunkAIMode,
paragraphChunkDeep: chunkSettings?.paragraphChunkDeep || defaultFormData.paragraphChunkDeep,
paragraphChunkMinSize:
chunkSettings?.paragraphChunkMinSize || defaultFormData.paragraphChunkMinSize,
paragraphChunkMaxSize:
chunkSettings?.paragraphChunkMaxSize || defaultFormData.paragraphChunkMaxSize,
chunkSize: chunkSettings?.chunkSize || defaultFormData.chunkSize,
chunkSplitter: chunkSettings?.chunkSplitter || defaultFormData.chunkSplitter,
indexSize: chunkSettings?.indexSize || defaultFormData.indexSize,
chunkSplitter: chunkSettings?.chunkSplitter || '',
qaPrompt: chunkSettings?.qaPrompt || Prompt_AgentQA.description
}
});