gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: mix search weight (#4170)

Browse Source 
* feat: mix search weight

* feat: svg render
This commit is contained in:
Archer
2025-03-14 18:31:37 +08:00
committed by GitHub
parent 6aa5e2c200
commit 93f7747904
29 changed files with 179 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
projects/app/src/components/Markdown/index.tsx
View File

@@ -85,7 +85,7 @@ export default React.memo(Markdown);
function Code(e: any) {
const { className, codeBlock, children } = e;
const match = /language-(\w+)/.exec(className || '');
const codeType = match?.[1];
const codeType = match?.[1]?.toLowerCase();
const strChildren = String(children);
@@ -105,7 +105,7 @@ function Code(e: any) {
if (codeType === CodeClassNameEnum.iframe) {
return <IframeCodeBlock code={strChildren} />;
}
if (codeType && codeType.toLowerCase() === CodeClassNameEnum.html) {
if (codeType === CodeClassNameEnum.html || codeType === CodeClassNameEnum.svg) {
return (
<IframeHtmlCodeBlock className={className} codeBlock={codeBlock} match={match}>
{children}

1
projects/app/src/components/Markdown/utils.ts
View File

@@ -8,6 +8,7 @@ export enum CodeClassNameEnum {
latex = 'latex',
iframe = 'iframe',
html = 'html',
svg = 'svg',
video = 'video',
audio = 'audio'
}

70
projects/app/src/components/core/app/DatasetParamsModal.tsx
View File

@@ -1,5 +1,17 @@
import React, { useEffect, useMemo, useState } from 'react';
import { Box, Button, Flex, HStack, ModalBody, ModalFooter, Switch } from '@chakra-ui/react';
import {
Box,
Button,
Flex,
HStack,
ModalBody,
ModalFooter,
Switch,
Slider,
SliderTrack,
SliderFilledTrack,
SliderThumb
} from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
import MyModal from '@fastgpt/web/components/common/MyModal';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
@@ -17,6 +29,7 @@ import { defaultDatasetMaxTokens } from '@fastgpt/global/core/app/constants';
import InputSlider from '@fastgpt/web/components/common/MySlider/InputSlider';
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import { AppDatasetSearchParamsType } from '@fastgpt/global/core/app/type';
import MyIcon from '@fastgpt/web/components/common/Icon';
enum SearchSettingTabEnum {
searchMode = 'searchMode',
@@ -28,6 +41,7 @@ const DatasetParamsModal = ({
searchMode = DatasetSearchModeEnum.embedding,
limit,
similarity,
embeddingWeight,
usingReRank,
rerankModel,
rerankWeight,
@@ -62,12 +76,13 @@ const DatasetParamsModal = ({
const { register, setValue, getValues, handleSubmit, watch } =
useForm<AppDatasetSearchParamsType>({
defaultValues: {
limit,
similarity,
searchMode,
embeddingWeight: embeddingWeight || 0.5,
usingReRank: !!usingReRank && teamPlanStatus?.standardConstants?.permissionReRank !== false,
rerankModel: rerankModel || defaultModels?.rerank?.model,
rerankWeight: rerankWeight || 0.5,
limit,
similarity,
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel: datasetSearchExtensionModel || defaultModels.llm?.model,
datasetSearchExtensionBg
@@ -75,6 +90,12 @@ const DatasetParamsModal = ({
});
const searchModeWatch = watch('searchMode');
const embeddingWeightWatch = watch('embeddingWeight');
const fullTextWeightWatch = useMemo(() => {
const val = 1 - (embeddingWeightWatch || 0.5);
return Number(val.toFixed(2));
}, [embeddingWeightWatch]);
const datasetSearchUsingCfrForm = watch('datasetSearchUsingExtensionQuery');
const queryExtensionModel = watch('datasetSearchExtensionModel');
@@ -168,8 +189,45 @@ const DatasetParamsModal = ({
{
title: t('common:core.dataset.search.mode.mixedRecall'),
desc: t('common:core.dataset.search.mode.mixedRecall desc'),
value: DatasetSearchModeEnum.mixedRecall
// children: searchModeWatch === DatasetSearchModeEnum.mixedRecall && <Box>111</Box>
value: DatasetSearchModeEnum.mixedRecall,
children: searchModeWatch === DatasetSearchModeEnum.mixedRecall && (
<Box mt={3}>
<HStack justifyContent={'space-between'}>
<Flex alignItems={'center'}>
<Box fontSize={'sm'} color={'myGray.900'}>
{t('common:core.dataset.search.mode.embedding')}
</Box>
<Box fontSize={'xs'} color={'myGray.500'}>
{embeddingWeightWatch}
</Box>
</Flex>
<Flex alignItems={'center'}>
<Box fontSize={'sm'} color={'myGray.900'}>
{t('common:core.dataset.search.score.fullText')}
</Box>
<Box fontSize={'xs'} color={'myGray.500'}>
{fullTextWeightWatch}
</Box>
</Flex>
</HStack>
<Slider
defaultValue={embeddingWeightWatch}
min={0.1}
max={0.9}
step={0.01}
onChange={(e) => {
setValue('embeddingWeight', Number(e.toFixed(2)));
}}
>
<SliderTrack bg={'#F9518E'}>
<SliderFilledTrack bg={'#3370FF'} />
</SliderTrack>
<SliderThumb boxShadow={'none'} bg={'none'}>
<MyIcon transform={'translateY(10px)'} name={'sliderTag'} w={'1rem'} />
</SliderThumb>
</Slider>
</Box>
)
}
]}
value={searchModeWatch}
@@ -201,7 +259,7 @@ const DatasetParamsModal = ({
<>
<HStack mt={3} justifyContent={'space-between'}>
<Box fontSize={'sm'} flex={'0 0 100px'} color={'myGray.700'}>
{t('app:rerank_weight')}
{t('common:rerank_weight')}
</Box>
<Box flex={'1 0 0'}>
<InputSlider

31
projects/app/src/components/core/chat/components/WholeResponseModal.tsx
View File

@@ -228,10 +228,23 @@ export const WholeResponseContent = ({
{activeModule?.searchMode && (
<Row
label={t('common:core.dataset.search.search mode')}
// @ts-ignore
value={t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
rawDom={
<Flex border={'base'} borderRadius={'md'} p={2}>
<Box>
{/* @ts-ignore */}
{t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
</Box>
{activeModule.embeddingWeight && (
<>{`(${t('chat:response_hybrid_weight', {
emb: activeModule.embeddingWeight,
text: 1 - activeModule.embeddingWeight
})})`}</>
)}
</Flex>
}
/>
)}
<Row
label={t('common:core.chat.response.module similarity')}
value={activeModule?.similarity}
@@ -239,7 +252,19 @@ export const WholeResponseContent = ({
<Row label={t('common:core.chat.response.module limit')} value={activeModule?.limit} />
<Row
label={t('common:core.chat.response.search using reRank')}
value={`${activeModule?.searchUsingReRank}`}
rawDom={
<Box border={'base'} borderRadius={'md'} p={2}>
{activeModule?.searchUsingReRank ? (
activeModule?.rerankModel ? (
<Box>{`${activeModule.rerankModel}: ${activeModule.rerankWeight}`}</Box>
) : (
'True'
)
) : (
`False`
)}
</Box>
}
/>
{activeModule.queryExtensionResult && (
<>

5
projects/app/src/global/core/dataset/api.d.ts vendored
View File

@@ -63,8 +63,13 @@ export type SearchTestProps = {
text: string;
[NodeInputKeyEnum.datasetSimilarity]?: number;
[NodeInputKeyEnum.datasetMaxTokens]?: number;
[NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
[NodeInputKeyEnum.datasetSearchRerankModel]?: string;
[NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
[NodeInputKeyEnum.datasetSearchUsingExtensionQuery]?: boolean;
[NodeInputKeyEnum.datasetSearchExtensionModel]?: string;

1
projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/SelectDatasetParams.tsx
View File

@@ -24,6 +24,7 @@ const SelectDatasetParam = ({ inputs = [], nodeId }: RenderInputProps) => {
const [data, setData] = useState<AppDatasetSearchParamsType>({
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
limit: 3000,
similarity: 0.5,
usingReRank: false,

17
projects/app/src/pageComponents/dataset/detail/Test.tsx
View File

@@ -36,9 +36,14 @@ type FormType = {
inputText: string;
searchParams: {
searchMode: `${DatasetSearchModeEnum}`;
embeddingWeight?: number;
usingReRank?: boolean;
rerankModel?: string;
rerankWeight?: number;
similarity?: number;
limit?: number;
usingReRank?: boolean;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
@@ -53,7 +58,6 @@ const Test = ({ datasetId }: { datasetId: string }) => {
const { pushDatasetTestItem } = useSearchTestStore();
const [inputType, setInputType] = useState<'text' | 'file'>('text');
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
const [refresh, setRefresh] = useState(false);
const [isFocus, setIsFocus] = useState(false);
const { File, onOpen } = useSelectFile({
fileType: '.csv',
@@ -66,7 +70,10 @@ const Test = ({ datasetId }: { datasetId: string }) => {
inputText: '',
searchParams: {
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false,
rerankModel: defaultModels?.rerank?.model,
rerankWeight: 0.5,
limit: 5000,
similarity: 0,
datasetSearchUsingExtensionQuery: false,
@@ -77,6 +84,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
});
const searchModeData = DatasetSearchModeMap[getValues(`searchParams.searchMode`)];
const searchParams = getValues('searchParams');
const {
isOpen: isOpenSelectMode,
@@ -294,15 +302,14 @@ const Test = ({ datasetId }: { datasetId: string }) => {
{isOpenSelectMode && (
<DatasetParamsModal
{...getValues('searchParams')}
{...searchParams}
maxTokens={20000}
onClose={onCloseSelectMode}
onSuccess={(e) => {
setValue('searchParams', {
...getValues('searchParams'),
...searchParams,
...e
});
setRefresh((state) => !state);
}}
/>
)}

10
projects/app/src/pages/api/core/dataset/searchTest.ts
View File

@@ -16,6 +16,7 @@ import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { getRerankModel } from '@fastgpt/service/core/ai/model';
async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTestResponse> {
const {
@@ -24,7 +25,11 @@ async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTes
limit = 1500,
similarity,
searchMode,
embeddingWeight,
usingReRank,
rerankModel,
rerankWeight,
datasetSearchUsingExtensionQuery = false,
datasetSearchExtensionModel,
@@ -63,7 +68,10 @@ async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTes
similarity,
datasetIds: [datasetId],
searchMode,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId))
embeddingWeight,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
rerankModel: getRerankModel(rerankModel),
rerankWeight
};
const { searchRes, tokens, queryExtensionResult, deepSearchResult, ...result } = datasetDeepSearch
? await deepRagSearch({

7
projects/app/src/web/core/app/utils.ts
View File

@@ -268,6 +268,13 @@ export function form2AppWorkflow(
valueType: WorkflowIOValueTypeEnum.string,
value: formData.dataset.searchMode
},
{
key: NodeInputKeyEnum.datasetSearchEmbeddingWeight,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.number,
value: formData.dataset.embeddingWeight
},
{
key: NodeInputKeyEnum.datasetSearchUsingReRank,
renderTypeList: [FlowNodeInputTypeEnum.hidden],