V4.9.1 feature (#4206)

* fix: remove DefaultTeam (#4037)

* fix :Get application bound knowledge base information logical rewrite (#4057)

* fix :Get application bound knowledge base information logical rewrite

* fix :Get application bound knowledge base information logical rewrite

* fix :Get application bound knowledge base information logical rewrite

* fix :Get application bound knowledge base information logical rewrite

* update package

* fix: import dataset step error;perf: ai proxy avatar (#4074)

* perf: pg config params

* perf: ai proxy avatar

* fix: import dataset step error

* feat: data input ux

* perf: app dataset rewite

* fix: 文本提取不支持arrayString,arrayNumber等jsonSchema (#4079)

* update doc ;perf: model test (#4098)

* perf: extract array

* update doc

* perf: model test

* perf: model test

* perf: think tag parse (#4102)

* chat quote reader (#3912)

* init chat quote full text reader

* linked structure

* dataset data linked

* optimize code

* fix ts build

* test finish

* delete log

* fix

* fix ts

* fix ts

* remove nextId

* initial scroll

* fix

* fix

* perf: chunk read   (#4109)

* package

* perf: chunk read

* feat: api dataset support pdf parse;fix: chunk reader auth (#4117)

* feat: api dataset support pdf parse

* fix: chunk reader auth

* feat: invitation link (#3979)

* feat: invitation link schema and apis

* feat: add invitation link

* feat: member status: active, leave, forbidden

* fix: expires show hours and minutes

* feat: invalid invitation link hint

* fix: typo

* chore: fix typo & i18n

* fix

* pref: fe

* feat: add ttl index for 30-day-clean-up

* perf: invite member code (#4118)

* perf: invite member code

* fix: ts

* fix: model test channel id;fix: quote reader (#4123)

* fix: model test channel id

* fix: quote reader

* fix chat quote reader (#4125)

* perf: model test;perf: sidebar trigger (#4127)

* fix: import dataset step error;perf: ai proxy avatar (#4074)

* perf: pg config params

* perf: ai proxy avatar

* fix: import dataset step error

* feat: data input ux

* perf: app dataset rewite

* perf: model test

* perf: sidebar trigger

* lock

* update nanoid version

* fix: select component ux

* fix: ts

* fix: vitest

* remove test

* fix: prompt toolcall ui (#4139)

* load log error adapt

* fix: prompt toolcall ui

* perf: commercial function tip

* update package

* pref: copy link (#4147)

* fix(i18n): namespace (#4143)

* hiden dataset source (#4152)

* hiden dataset source

* perf: reader

* chore: move all tests into a single folder (#4160)

* fix modal close scroll (#4162)

* fix modal close scroll

* update refresh

* feat: rerank modal select and weight (#4164)

* fix loadInitData refresh (#4169)

* fix

* fix

* form input number default & api dataset max token

* feat: mix search weight (#4170)

* feat: mix search weight

* feat: svg render

* fix: avatar error remove (#4173)

* fix: avatar error remove

* fix: index

* fix: guide

* fix: auth

* update package;fix: input data model ui (#4181)

* update package

* fix: ts

* update config

* update jieba package

* add type sign

* fix: input data ui

* fix: page title refresh (#4186)

* fix: ts

* update jieba package

* fix: page title refresh

* fix: remove member length check when opening invite create modal (#4193)

* add env to check internal ip (#4187)

* fix: ts

* update jieba package

* add env to check internal ip

* package

* fix: jieba

* reset package

* update config

* fix: jieba package

* init shell

* init version

* change team reload

* update jieba package (#4200)

* update jieba package

* package

* update package

* remove invalid code

* action

* package (#4201)

* package

* update package

* remove invalid code

* package

* remove i18n tip (#4202)

* doc (#4205)

* fix: i18n (#4208)

* fix: next config (#4207)

* reset package

* i18n

* update config

* i18n

* remove log

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>
Co-authored-by: shilin <39396378+shilin66@users.noreply.github.com>
Co-authored-by: heheer <heheer@sealos.io>
This commit is contained in:
Archer
2025-03-18 14:40:41 +08:00
committed by GitHub
parent 56793114d8
commit e75d81d05a
316 changed files with 10626 additions and 8464 deletions

View File

@@ -60,7 +60,7 @@ const ApiDatasetForm = ({
<Input
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
maxLength={200}
maxLength={2000}
{...register('apiServer.authorization')}
/>
</Flex>

View File

@@ -5,7 +5,6 @@ import {
delOneDatasetDataById,
getDatasetCollectionById
} from '@/web/core/dataset/api';
import { useQuery } from '@tanstack/react-query';
import { useToast } from '@fastgpt/web/hooks/useToast';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useConfirm } from '@fastgpt/web/hooks/useConfirm';

View File

@@ -113,7 +113,7 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
],
[ImportDataSourceEnum.fileLink]: [
{
title: t('dataset:import_select_file')
title: t('dataset:import_select_link')
},
{
title: t('dataset:import_param_setting')

View File

@@ -36,19 +36,19 @@ import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import { shadowLight } from '@fastgpt/web/styles/theme';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import { useToast } from '@fastgpt/web/hooks/useToast';
function DataProcess() {
const { t } = useTranslation();
const { feConfigs } = useSystemStore();
const { toast } = useToast();
const { goToNext, processParamsForm, chunkSizeField, minChunkSize, maxChunkSize } =
useContextSelector(DatasetImportContext, (v) => v);
const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
const { getValues, setValue, register, watch } = processParamsForm;
const { setValue, register, watch } = processParamsForm;
const trainingType = watch('trainingType');
const chunkSettingMode = watch('chunkSettingMode');
const qaPrompt = watch('qaPrompt');
const {
isOpen: isOpenCustomPrompt,
@@ -65,7 +65,7 @@ function DataProcess() {
value: key as DatasetCollectionDataProcessModeEnum,
tooltip: t(value.tooltip as any)
}));
}, []);
}, [t]);
const Title = useCallback(({ title }: { title: string }) => {
return (
@@ -159,23 +159,36 @@ function DataProcess() {
gridTemplateColumns={'repeat(2, 1fr)'}
/>
</Box>
{trainingType === DatasetCollectionDataProcessModeEnum.chunk && feConfigs?.isPlus && (
{trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
<Box mt={6}>
<Box fontSize={'sm'} mb={2} color={'myGray.600'}>
{t('dataset:enhanced_indexes')}
</Box>
<HStack gap={[3, 7]}>
<HStack flex={'1'} spacing={1}>
<Checkbox {...register('autoIndexes')}>
<FormLabel>{t('dataset:auto_indexes')}</FormLabel>
</Checkbox>
<MyTooltip
label={!feConfigs?.isPlus ? t('common:commercial_function_tip') : ''}
>
<Checkbox isDisabled={!feConfigs?.isPlus} {...register('autoIndexes')}>
<FormLabel>{t('dataset:auto_indexes')}</FormLabel>
</Checkbox>
</MyTooltip>
<QuestionTip label={t('dataset:auto_indexes_tips')} />
</HStack>
<HStack flex={'1'} spacing={1}>
<MyTooltip
label={!datasetDetail?.vlmModel ? t('common:error_vlm_not_config') : ''}
label={
!feConfigs?.isPlus
? t('common:commercial_function_tip')
: !datasetDetail?.vlmModel
? t('common:error_vlm_not_config')
: ''
}
>
<Checkbox isDisabled={!datasetDetail?.vlmModel} {...register('imageIndex')}>
<Checkbox
isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
{...register('imageIndex')}
>
<FormLabel>{t('dataset:image_auto_parse')}</FormLabel>
</Checkbox>
</MyTooltip>
@@ -271,7 +284,7 @@ function DataProcess() {
}
}}
>
{getValues('qaPrompt')}
{qaPrompt}
<Box
display={'none'}
@@ -320,44 +333,6 @@ function DataProcess() {
</AccordionPanel>
</AccordionItem>
{/* <AccordionItem mt={4} border={'none'}>
<Title title={t('dataset:import_model_config')} />
<AccordionPanel p={2} fontSize={'sm'}>
<Box>
<Box>{t('common:core.ai.model.Dataset Agent Model')}</Box>
<Box mt={1}>
<AIModelSelector
w={'100%'}
value={llmModel}
list={datasetModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
setValue('llmModel', e);
}}
/>
</Box>
</Box>
<Box pt={5}>
<Box>{t('dataset:vllm_model')}</Box>
<Box mt={1}>
<AIModelSelector
w={'100%'}
value={vlmModel}
list={vllmModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
setValue('vlmModel', e);
}}
/>
</Box>
</Box>
</AccordionPanel>
</AccordionItem> */}
<Flex mt={5} gap={3} justifyContent={'flex-end'}>
<Button
onClick={() => {
@@ -372,7 +347,7 @@ function DataProcess() {
{isOpenCustomPrompt && (
<PromptTextarea
defaultValue={getValues('qaPrompt')}
defaultValue={qaPrompt}
onChange={(e) => {
setValue('qaPrompt', e);
}}

View File

@@ -21,6 +21,7 @@ const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
const APIDatasetCollection = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
@@ -29,7 +30,8 @@ const APIDatasetCollection = () => {
<>
{activeStep === 0 && <CustomAPIFileInput />}
{activeStep === 1 && <DataProcess />}
{activeStep === 2 && <Upload />}
{activeStep === 2 && <PreviewData />}
{activeStep === 3 && <Upload />}
</>
);
};

View File

@@ -27,6 +27,7 @@ const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
const ExternalFileCollection = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
@@ -35,7 +36,8 @@ const ExternalFileCollection = () => {
<>
{activeStep === 0 && <CustomLinkInput />}
{activeStep === 1 && <DataProcess />}
{activeStep === 2 && <Upload />}
{activeStep === 2 && <PreviewData />}
{activeStep === 3 && <Upload />}
</>
);
};

View File

@@ -13,6 +13,7 @@ const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
const CustomTet = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
@@ -20,7 +21,8 @@ const CustomTet = () => {
<>
{activeStep === 0 && <CustomTextInput />}
{activeStep === 1 && <DataProcess />}
{activeStep === 2 && <Upload />}
{activeStep === 2 && <PreviewData />}
{activeStep === 3 && <Upload />}
</>
);
};

View File

@@ -16,6 +16,7 @@ const DataProcess = dynamic(() => import('../commonProgress/DataProcess'), {
loading: () => <Loading fixed={false} />
});
const Upload = dynamic(() => import('../commonProgress/Upload'));
const PreviewData = dynamic(() => import('../commonProgress/PreviewData'));
const LinkCollection = () => {
const activeStep = useContextSelector(DatasetImportContext, (v) => v.activeStep);
@@ -24,7 +25,8 @@ const LinkCollection = () => {
<>
{activeStep === 0 && <CustomLinkImport />}
{activeStep === 1 && <DataProcess />}
{activeStep === 2 && <Upload />}
{activeStep === 2 && <PreviewData />}
{activeStep === 3 && <Upload />}
</>
);
};

View File

@@ -57,7 +57,7 @@ const ReTraining = () => {
qaChunkSize: collection.chunkSize,
customSplitChar: collection.chunkSplitter,
qaPrompt: collection.qaPrompt,
webSelector: collection.metadata?.webSelector
webSelector: collection.metadata?.webPageSelector
});
}
});

View File

@@ -195,7 +195,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
label: item.name,
value: item.model
}))}
onchange={(e) => {
onChange={(e) => {
const vectorModel = embeddingModelList.find((item) => item.model === e);
if (!vectorModel) return;
return onOpenConfirmRebuild(async () => {
@@ -220,7 +220,7 @@ const Info = ({ datasetId }: { datasetId: string }) => {
value: item.model
}))}
fontSize={'mini'}
onchange={(e) => {
onChange={(e) => {
const agentModel = datasetModelList.find((item) => item.model === e);
if (!agentModel) return;
setValue('agentModel', agentModel);
@@ -230,30 +230,28 @@ const Info = ({ datasetId }: { datasetId: string }) => {
</Box>
</Box>
{feConfigs?.isPlus && (
<Box pt={5}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:vllm_model')}
</FormLabel>
<Box pt={2}>
<AIModelSelector
w={'100%'}
value={vlmModel?.model}
list={vllmModelList.map((item) => ({
label: item.name,
value: item.model
}))}
fontSize={'mini'}
onchange={(e) => {
const vlmModel = vllmModelList.find((item) => item.model === e);
if (!vlmModel) return;
setValue('vlmModel', vlmModel);
return handleSubmit((data) => onSave({ ...data, vlmModel }))();
}}
/>
</Box>
<Box pt={5}>
<FormLabel fontSize={'mini'} fontWeight={'500'}>
{t('dataset:vllm_model')}
</FormLabel>
<Box pt={2}>
<AIModelSelector
w={'100%'}
value={vlmModel?.model}
list={vllmModelList.map((item) => ({
label: item.name,
value: item.model
}))}
fontSize={'mini'}
onChange={(e) => {
const vlmModel = vllmModelList.find((item) => item.model === e);
if (!vlmModel) return;
setValue('vlmModel', vlmModel);
return handleSubmit((data) => onSave({ ...data, vlmModel }))();
}}
/>
</Box>
)}
</Box>
{feConfigs?.isPlus && (
<Flex alignItems={'center'} pt={5}>

View File

@@ -1,12 +1,6 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Box, Flex, Button, Textarea } from '@chakra-ui/react';
import {
FieldArrayWithId,
UseFieldArrayRemove,
UseFormRegister,
useFieldArray,
useForm
} from 'react-hook-form';
import { Box, Flex, Button, Textarea, ModalFooter, HStack, VStack } from '@chakra-ui/react';
import { UseFormRegister, useFieldArray, useForm } from 'react-hook-form';
import {
postInsertData2Dataset,
putDatasetDataById,
@@ -17,36 +11,36 @@ import { useToast } from '@fastgpt/web/hooks/useToast';
import MyIcon from '@fastgpt/web/components/common/Icon';
import MyModal from '@fastgpt/web/components/common/MyModal';
import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
import { useQuery } from '@tanstack/react-query';
import { useTranslation } from 'next-i18next';
import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
import DeleteIcon from '@fastgpt/web/components/common/Icon/delete';
import { defaultCollectionDetail } from '@/web/core/dataset/constants';
import { getDocPath } from '@/web/common/system/doc';
import MyBox from '@fastgpt/web/components/common/MyBox';
import { getErrText } from '@fastgpt/global/common/error/utils';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import LightRowTabs from '@fastgpt/web/components/common/Tabs/LightRowTabs';
import styles from './styles.module.scss';
import {
DatasetDataIndexTypeEnum,
getDatasetIndexMapData
} from '@fastgpt/global/core/dataset/data/constants';
import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyIconButton from '@fastgpt/web/components/common/Icon/button';
export type InputDataType = {
q: string;
a: string;
indexes: (Omit<DatasetDataIndexItemType, 'dataId'> & {
dataId?: string; // pg data id
fold: boolean;
})[];
};
enum TabEnum {
content = 'content',
index = 'index'
chunk = 'chunk',
qa = 'qa'
}
const InputDataModal = ({
@@ -64,73 +58,47 @@ const InputDataModal = ({
}) => {
const { t } = useTranslation();
const { toast } = useToast();
const [currentTab, setCurrentTab] = useState(TabEnum.content);
const { embeddingModelList, defaultModels } = useSystemStore();
const [currentTab, setCurrentTab] = useState(TabEnum.chunk);
const { register, handleSubmit, reset, control } = useForm<InputDataType>();
const {
fields: indexes,
append: appendIndexes,
remove: removeIndexes
prepend: prependIndexes,
remove: removeIndexes,
update: updateIndexes
} = useFieldArray({
control,
name: 'indexes'
});
const tabList = [
{
label: (
<Flex align={'center'}>
<Box>{t('common:dataset.data.edit.divide_content')}</Box>
</Flex>
),
value: TabEnum.content
},
{
label: (
<Flex align={'center'}>
<Box>{t('common:dataset.data.edit.Index', { amount: indexes.length })}</Box>
<MyTooltip label={t('common:core.app.tool_label.view_doc')}>
<MyIcon
name={'book'}
w={'1rem'}
mr={'0.38rem'}
color={'myGray.500'}
ml={1}
onClick={() =>
window.open(getDocPath('/docs/guide/knowledge_base/dataset_engine/'), '_blank')
}
_hover={{
color: 'primary.600',
cursor: 'pointer'
}}
/>
</MyTooltip>
</Flex>
),
value: TabEnum.index
}
];
const { data: collection = defaultCollectionDetail } = useQuery(
['loadCollectionId', collectionId],
const { data: collection = defaultCollectionDetail } = useRequest2(
() => {
return getDatasetCollectionById(collectionId);
},
{
manual: false,
refreshDeps: [collectionId]
}
);
const { isFetching: isFetchingData } = useQuery(
['getDatasetDataItemById', dataId],
() => {
const { loading: isFetchingData } = useRequest2(
async () => {
if (dataId) return getDatasetDataItemById(dataId);
return null;
},
{
manual: false,
refreshDeps: [dataId],
onSuccess(res) {
if (res) {
reset({
q: res.q,
a: res.a,
indexes: res.indexes
indexes: res.indexes.map((item) => ({
...item,
fold: true
}))
});
} else if (defaultValue) {
reset({
@@ -138,6 +106,10 @@ const InputDataModal = ({
a: defaultValue.a
});
}
if (res?.a || defaultValue?.a) {
setCurrentTab(TabEnum.qa);
}
},
onError(err) {
toast({
@@ -161,7 +133,6 @@ const InputDataModal = ({
const { runAsync: sureImportData, loading: isImporting } = useRequest2(
async (e: InputDataType) => {
if (!e.q) {
setCurrentTab(TabEnum.content);
return Promise.reject(t('common:dataset.data.input is empty'));
}
@@ -175,9 +146,9 @@ const InputDataModal = ({
const dataId = await postInsertData2Dataset({
collectionId: collection._id,
q: e.q,
a: e.a,
a: currentTab === TabEnum.qa ? e.a : '',
// Contains no default index
indexes: e.indexes
indexes: e.indexes.filter((item) => !!item.text?.trim())
});
return {
@@ -186,6 +157,7 @@ const InputDataModal = ({
};
},
{
refreshDeps: [currentTab],
successToast: t('common:dataset.data.Input Success Tip'),
onSuccess(e) {
reset({
@@ -205,11 +177,11 @@ const InputDataModal = ({
async (e: InputDataType) => {
if (!dataId) return Promise.reject(t('common:common.error.unKnow'));
// not exactly same
await putDatasetDataById({
dataId,
...e,
indexes: e.indexes
q: e.q,
a: currentTab === TabEnum.qa ? e.a : '',
indexes: e.indexes.filter((item) => !!item.text?.trim())
});
return {
@@ -218,6 +190,7 @@ const InputDataModal = ({
};
},
{
refreshDeps: [currentTab],
successToast: t('common:dataset.data.Update Success Tip'),
onSuccess(data) {
onSuccess(data);
@@ -267,49 +240,174 @@ const InputDataModal = ({
isLoading={isLoading}
h={'100%'}
py={[6, '1.5rem']}
px={[5, '3.25rem']}
>
<Flex justify={'space-between'} gap={4} w={'100%'}>
<Flex justify={'space-between'} pb={4}>
<LightRowTabs<TabEnum>
list={tabList}
p={0}
value={currentTab}
onChange={(e: TabEnum) => setCurrentTab(e)}
/>
</Flex>
{currentTab === TabEnum.index && (
<Button
variant={'whiteBase'}
boxShadow={'1'}
p={0}
onClick={() =>
appendIndexes({
type: DatasetDataIndexTypeEnum.custom,
text: ''
})
}
>
<Flex px={'0.62rem'} py={2}>
<MyIcon name={'common/addLight'} w={'1rem'} mr={'0.38rem'} />
{t('common:add_new')}
</Flex>
</Button>
)}
</Flex>
<Box w={'100%'} flexGrow={1} overflow={'scroll'}>
{currentTab === TabEnum.content && <InputTab maxToken={maxToken} register={register} />}
{currentTab === TabEnum.index && (
<DataIndex
register={register}
maxToken={maxToken}
removeIndexes={removeIndexes}
indexes={indexes}
/>
)}
{/* Tab */}
<Box px={[5, '3.25rem']}>
<FillRowTabs
list={[
{ label: t('common:dataset_data_input_chunk'), value: TabEnum.chunk },
{ label: t('common:dataset_data_input_qa'), value: TabEnum.qa }
]}
py={1}
value={currentTab}
onChange={(e) => {
setCurrentTab(e);
}}
/>
</Box>
<Flex justifyContent={'flex-end'} pt={8} pb={[8, 0]} h={[24, 16]}>
<Flex flex={'1 0 0'} h={['auto', '0']} gap={6} flexDir={['column', 'row']} px={[5, '0']}>
{/* Data */}
<Flex
pt={4}
pl={[0, '3.25rem']}
flexDir={'column'}
h={'100%'}
gap={3}
flex={'1 0 0'}
w={['100%', 0]}
overflow={['unset', 'auto']}
>
<Flex flexDir={'column'} h={'100%'}>
<FormLabel required mb={1} h={'30px'}>
{currentTab === TabEnum.chunk
? t('common:dataset_data_input_chunk_content')
: t('common:dataset_data_input_q')}
</FormLabel>
<Textarea
resize={'none'}
placeholder={t('common:dataset_data_import_q_placeholder', { maxToken })}
className={styles.scrollbar}
maxLength={maxToken}
flex={'1 0 0'}
tabIndex={1}
_focus={{
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
bg={'myGray.25'}
borderRadius={'md'}
borderColor={'myGray.200'}
{...register(`q`, {
required: true
})}
/>
</Flex>
{currentTab === TabEnum.qa && (
<Flex flexDir={'column'} h={'100%'}>
<FormLabel required mb={1}>
{t('common:dataset_data_input_a')}
</FormLabel>
<Textarea
resize={'none'}
placeholder={t('common:dataset_data_import_q_placeholder', { maxToken })}
className={styles.scrollbar}
flex={'1 0 0'}
tabIndex={1}
bg={'myGray.25'}
maxLength={maxToken}
borderRadius={'md'}
border={'1.5px solid '}
borderColor={'myGray.200'}
{...register('a', { required: true })}
/>
</Flex>
)}
</Flex>
{/* Index */}
<Box
pt={4}
pr={[0, '3.25rem']}
flex={'1 0 0'}
w={['100%', 0]}
overflow={['unset', 'auto']}
>
<Flex alignItems={'flex-start'} justifyContent={'space-between'} h={'30px'}>
<FormLabel>
{t('common:dataset.data.edit.Index', {
amount: indexes.length
})}
</FormLabel>
<Button
variant={'whiteBase'}
size={'sm'}
p={0}
transform={'translateY(-6px)'}
onClick={() =>
prependIndexes({
type: DatasetDataIndexTypeEnum.custom,
text: '',
fold: false
})
}
>
<Flex px={'0.62rem'} py={2}>
<MyIcon name={'common/addLight'} w={'1rem'} mr={'0.38rem'} />
{t('common:add_new')}
</Flex>
</Button>
</Flex>
<VStack>
{indexes?.map((index, i) => {
const data = getDatasetIndexMapData(index.type);
return (
<Box
key={index.dataId || i}
p={4}
borderRadius={'md'}
border={'base'}
bg={'myGray.25'}
w={'100%'}
_hover={{
'& .delete': {
display: 'block'
}
}}
>
{/* Header */}
<Flex mb={2} alignItems={'center'}>
<FormLabel flex={'1 0 0'}>{t(data.label)}</FormLabel>
{/* Delete */}
{index.type !== 'default' && (
<HStack className={'delete'} borderRight={'base'} pr={3} mr={2}>
<DeleteIcon
onClick={() => {
removeIndexes(i);
}}
/>
</HStack>
)}
{indexes.length > 1 && (
<MyIconButton
icon={index.fold ? 'core/chat/chevronDown' : 'core/chat/chevronUp'}
onClick={() => {
updateIndexes(i, { ...index, fold: !index.fold });
}}
/>
)}
</Flex>
{/* Content */}
<DataIndexTextArea
disabled={index.type === 'default'}
index={i}
value={index.text}
isFolder={index.fold && indexes.length > 1}
maxToken={maxToken}
register={register}
onFocus={() => {
updateIndexes(i, { ...index, fold: false });
}}
/>
</Box>
);
})}
</VStack>
</Box>
</Flex>
<ModalFooter px={[5, '3.25rem']} py={0} pt={4}>
<MyTooltip
label={collection.permission.hasWritePer ? '' : t('common:dataset.data.Can not edit')}
>
@@ -322,7 +420,7 @@ const InputDataModal = ({
{dataId ? t('common:common.Confirm Update') : t('common:common.Confirm Import')}
</Button>
</MyTooltip>
</Flex>
</ModalFooter>
</MyBox>
</MyModal>
);
@@ -330,153 +428,23 @@ const InputDataModal = ({
export default React.memo(InputDataModal);
const InputTab = ({
maxToken,
register
}: {
maxToken: number;
register: UseFormRegister<InputDataType>;
}) => {
const { t } = useTranslation();
return (
<>
<Flex h={'100%'} gap={6} flexDir={['column', 'row']} w={'100%'}>
<Flex flexDir={'column'} flex={1}>
<Flex mb={2} fontWeight={'medium'} fontSize={'sm'} alignItems={'center'} h={8}>
<Box color={'red.600'}>*</Box>
<Box color={'myGray.900'}>{t('common:core.dataset.data.Main Content')}</Box>
<QuestionTip label={t('common:core.dataset.data.Data Content Tip')} ml={1} />
</Flex>
<Box
borderRadius={'md'}
border={'1.5px solid var(--Gray-Modern-200, #E8EBF0)'}
bg={'myGray.25'}
flex={1}
>
<Textarea
resize={'none'}
placeholder={t('core.dataset.data.Data Content Placeholder', { maxToken })}
className={styles.scrollbar}
maxLength={maxToken}
h={'100%'}
tabIndex={1}
_focus={{
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
borderColor={'transparent'}
bg={'myGray.25'}
{...register(`q`, {
required: true
})}
/>
</Box>
</Flex>
<Flex flex={1} flexDir={'column'}>
<Flex mb={2} fontWeight={'medium'} fontSize={'sm'} alignItems={'center'} h={8}>
<Box color={'myGray.900'}>{t('common:core.dataset.data.Auxiliary Data')}</Box>
<QuestionTip label={t('common:core.dataset.data.Auxiliary Data Tip')} ml={1} />
</Flex>
<Box
borderRadius={'md'}
border={'1.5px solid '}
borderColor={'myGray.200'}
bg={'myGray.25'}
flex={1}
>
<Textarea
resize={'none'}
placeholder={t('core.dataset.data.Auxiliary Data Placeholder', {
maxToken: maxToken * 1.5
})}
className={styles.scrollbar}
borderColor={'transparent'}
h={'100%'}
tabIndex={1}
bg={'myGray.25'}
maxLength={maxToken * 1.5}
{...register('a')}
/>
</Box>
</Flex>
</Flex>
</>
);
};
const DataIndex = ({
maxToken,
register,
indexes,
removeIndexes
}: {
maxToken: number;
register: UseFormRegister<InputDataType>;
indexes: FieldArrayWithId<InputDataType, 'indexes', 'id'>[];
removeIndexes: UseFieldArrayRemove;
}) => {
const { t } = useTranslation();
return (
<>
<Flex mt={3} gap={3} flexDir={'column'}>
{indexes?.map((index, i) => {
const data = getDatasetIndexMapData(index.type);
return (
<Box
key={index.dataId || i}
p={4}
borderRadius={'md'}
border={'1.5px solid var(--Gray-Modern-200, #E8EBF0)'}
bg={'myGray.25'}
_hover={{
'& .delete': {
display: 'block'
}
}}
>
<Flex mb={2}>
<Box flex={1} fontWeight={'medium'} fontSize={'sm'} color={'myGray.900'}>
{t(data.label)}
</Box>
{index.type !== 'default' && (
<DeleteIcon
onClick={() => {
removeIndexes(i);
}}
/>
)}
</Flex>
<DataIndexTextArea
disabled={index.type === 'default'}
index={i}
value={index.text}
maxToken={maxToken}
register={register}
/>
</Box>
);
})}
</Flex>
</>
);
};
const textareaMinH = '40px';
const DataIndexTextArea = ({
value,
index,
maxToken,
register,
disabled
disabled,
isFolder,
onFocus
}: {
value: string;
index: number;
maxToken: number;
register: UseFormRegister<InputDataType>;
disabled?: boolean;
isFolder: boolean;
onFocus: () => void;
}) => {
const { t } = useTranslation();
const TextareaDom = useRef<HTMLTextAreaElement | null>(null);
@@ -501,41 +469,76 @@ const DataIndexTextArea = ({
}
}, []);
return disabled ? (
<Box fontSize={'sm'} color={'myGray.500'} whiteSpace={'pre-wrap'}>
{value}
const onclickMark = () => {
TextareaDom?.current?.focus();
onFocus();
};
return (
<Box
pos={'relative'}
{...(isFolder
? {
maxH: '50px',
overflow: 'hidden'
}
: {
maxH: 'auto'
})}
>
{disabled ? (
<Box fontSize={'sm'} color={'myGray.500'} whiteSpace={'pre-wrap'}>
{value}
</Box>
) : (
<Textarea
maxLength={maxToken}
borderColor={'transparent'}
className={styles.scrollbar}
minH={textareaMinH}
px={0}
pt={0}
isRequired={required}
whiteSpace={'pre-wrap'}
resize={'none'}
_focus={{
px: 3,
py: 1,
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
placeholder={t('common:dataset.data.Index Placeholder')}
ref={(e) => {
if (e) TextareaDom.current = e;
TextareaRef(e);
}}
required
name={name}
onChange={(e) => {
autoHeight(e);
onTextChange(e);
}}
onFocus={autoHeight}
onBlur={onBlur}
/>
)}
{isFolder && (
<Box
pos={'absolute'}
bottom={0}
left={0}
right={0}
top={0}
bg={'linear-gradient(182deg, rgba(251, 251, 252, 0.00) 1.76%, #FBFBFC 84.07%)'}
{...(disabled
? {}
: {
cursor: 'pointer',
onClick: onclickMark
})}
/>
)}
</Box>
) : (
<Textarea
maxLength={maxToken}
borderColor={'transparent'}
className={styles.scrollbar}
minH={textareaMinH}
px={0}
pt={0}
isRequired={required}
whiteSpace={'pre-wrap'}
resize={'none'}
_focus={{
px: 3,
py: 1,
borderColor: 'primary.500',
boxShadow: '0px 0px 0px 2.4px rgba(51, 112, 255, 0.15)',
bg: 'white'
}}
placeholder={t('common:dataset.data.Index Placeholder')}
ref={(e) => {
if (e) TextareaDom.current = e;
TextareaRef(e);
}}
required
name={name}
onChange={(e) => {
autoHeight(e);
onTextChange(e);
}}
onFocus={autoHeight}
onBlur={onBlur}
/>
);
};

View File

@@ -6,7 +6,6 @@ import { useRouter } from 'next/router';
import { useContextSelector } from 'use-context-selector';
import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
import LightRowTabs from '@fastgpt/web/components/common/Tabs/LightRowTabs';
import { useI18n } from '@/web/context/I18n';
import { useSystem } from '@fastgpt/web/hooks/useSystem';
import MyPopover from '@fastgpt/web/components/common/MyPopover';
import ParentPaths from '@/components/common/ParentPaths';
@@ -22,7 +21,6 @@ export enum TabEnum {
const NavBar = ({ currentTab }: { currentTab: TabEnum }) => {
const theme = useTheme();
const { t } = useTranslation();
const { datasetT } = useI18n();
const router = useRouter();
const query = router.query;
const { isPc } = useSystem();
@@ -168,7 +166,7 @@ const NavBar = ({ currentTab }: { currentTab: TabEnum }) => {
{rebuildingCount > 0 && (
<Box mb={3}>
<Box fontSize={'sm'}>
{datasetT('rebuilding_index_count', { count: rebuildingCount })}
{t('dataset:rebuilding_index_count', { count: rebuildingCount })}
</Box>
</Box>
)}

View File

@@ -36,9 +36,14 @@ type FormType = {
inputText: string;
searchParams: {
searchMode: `${DatasetSearchModeEnum}`;
embeddingWeight?: number;
usingReRank?: boolean;
rerankModel?: string;
rerankWeight?: number;
similarity?: number;
limit?: number;
usingReRank?: boolean;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
@@ -53,7 +58,6 @@ const Test = ({ datasetId }: { datasetId: string }) => {
const { pushDatasetTestItem } = useSearchTestStore();
const [inputType, setInputType] = useState<'text' | 'file'>('text');
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
const [refresh, setRefresh] = useState(false);
const [isFocus, setIsFocus] = useState(false);
const { File, onOpen } = useSelectFile({
fileType: '.csv',
@@ -66,7 +70,10 @@ const Test = ({ datasetId }: { datasetId: string }) => {
inputText: '',
searchParams: {
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false,
rerankModel: defaultModels?.rerank?.model,
rerankWeight: 0.5,
limit: 5000,
similarity: 0,
datasetSearchUsingExtensionQuery: false,
@@ -77,6 +84,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
});
const searchModeData = DatasetSearchModeMap[getValues(`searchParams.searchMode`)];
const searchParams = getValues('searchParams');
const {
isOpen: isOpenSelectMode,
@@ -186,7 +194,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
// }
]}
value={inputType}
onchange={(e) => setInputType(e)}
onChange={(e) => setInputType(e)}
/>
<Button
@@ -294,15 +302,14 @@ const Test = ({ datasetId }: { datasetId: string }) => {
{isOpenSelectMode && (
<DatasetParamsModal
{...getValues('searchParams')}
{...searchParams}
maxTokens={20000}
onClose={onCloseSelectMode}
onSuccess={(e) => {
setValue('searchParams', {
...getValues('searchParams'),
...searchParams,
...e
});
setRefresh((state) => !state);
}}
/>
)}

View File

@@ -205,7 +205,7 @@ const CreateModal = ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
onChange={(e) => {
setValue('vectorModel' as const, e);
}}
/>
@@ -237,45 +237,43 @@ const CreateModal = ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
onChange={(e) => {
setValue('agentModel', e);
}}
/>
</Box>
</Flex>
{feConfigs?.isPlus && (
<Flex
mt={6}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
<Flex
mt={6}
alignItems={['flex-start', 'center']}
justify={'space-between'}
flexDir={['column', 'row']}
>
<HStack
spacing={1}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
>
<HStack
spacing={1}
flex={['', '0 0 110px']}
fontSize={'sm'}
color={'myGray.900'}
fontWeight={500}
pb={['12px', '0']}
>
<Box>{t('dataset:vllm_model')}</Box>
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={vlmModel}
list={vllmModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onchange={(e) => {
setValue('vlmModel', e);
}}
/>
</Box>
</Flex>
)}
<Box>{t('dataset:vllm_model')}</Box>
</HStack>
<Box w={['100%', '300px']}>
<AIModelSelector
w={['100%', '300px']}
value={vlmModel}
list={vllmModelList.map((item) => ({
label: item.name,
value: item.model
}))}
onChange={(e) => {
setValue('vlmModel', e);
}}
/>
</Box>
</Flex>
{/* @ts-ignore */}
<ApiDatasetForm type={type} form={form} />