Compare commits

...

5 Commits

Author SHA1 Message Date
Archer
93f7747904 feat: mix search weight (#4170)
* feat: mix search weight

* feat: svg render
2025-03-14 18:31:37 +08:00
heheer
6aa5e2c200 fix loadInitData refresh (#4169)
* fix

* fix

* form input number default & api dataset max token
2025-03-14 18:07:47 +08:00
Archer
d8712d4092 feat: rerank modal select and weight (#4164) 2025-03-14 14:49:27 +08:00
heheer
561a496f80 fix modal close scroll (#4162)
* fix modal close scroll

* update refresh
2025-03-14 14:21:48 +08:00
Finley Ge
bd369d3b09 chore: move all tests into a single folder (#4160) 2025-03-14 11:39:44 +08:00
58 changed files with 460 additions and 210 deletions

View File

@@ -13,6 +13,8 @@ weight: 799
2. 知识库分块阅读器。
3. API 知识库支持 PDF 增强解析。
4. 邀请团队成员,改为邀请链接模式。
5. 支持混合检索权重设置。
6. 支持重排模型选择和权重设置,同时调整了知识库搜索权重计算方式,改成 搜索权重 + 重排权重,而不是向量检索权重+全文检索权重+重排权重。
## ⚙️ 优化

View File

@@ -12,7 +12,7 @@
"previewIcon": "node ./scripts/icon/index.js",
"api:gen": "tsc ./scripts/openapi/index.ts && node ./scripts/openapi/index.js && npx @redocly/cli build-docs ./scripts/openapi/openapi.json -o ./projects/app/public/openapi/index.html",
"create:i18n": "node ./scripts/i18n/index.js",
"test": "vitest run --exclude './projects/app/src/test/**'",
"test": "vitest run --exclude 'test/cases/spec'",
"test:all": "vitest run",
"test:workflow": "vitest run workflow"
},

View File

@@ -6,7 +6,7 @@ import type {
EmbeddingModelItemType,
AudioSpeechModels,
STTModelType,
ReRankModelItemType
RerankModelItemType
} from '../../../core/ai/model.d';
import { SubTypeEnum } from '../../../support/wallet/sub/constants';
@@ -35,7 +35,7 @@ export type FastGPTConfigFileType = {
// Abandon
llmModels?: ChatModelItemType[];
vectorModels?: EmbeddingModelItemType[];
reRankModels?: ReRankModelItemType[];
reRankModels?: RerankModelItemType[];
audioSpeechModels?: TTSModelType[];
whisperModel?: STTModelType;
};

View File

@@ -72,7 +72,7 @@ export type EmbeddingModelItemType = PriceType &
queryConfig?: Record<string, any>; // Custom parameters for query
};
export type ReRankModelItemType = PriceType &
export type RerankModelItemType = PriceType &
BaseModelItemType & {
type: ModelTypeEnum.rerank;
};

View File

@@ -71,6 +71,20 @@ export type AppDetailType = AppSchema & {
permission: AppPermission;
};
export type AppDatasetSearchParamsType = {
searchMode: `${DatasetSearchModeEnum}`;
limit?: number; // limit max tokens
similarity?: number;
embeddingWeight?: number; // embedding weight, fullText weight = 1 - embeddingWeight
usingReRank?: boolean;
rerankModel?: string;
rerankWeight?: number;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
};
export type AppSimpleEditFormType = {
// templateId: string;
aiSettings: {
@@ -88,14 +102,7 @@ export type AppSimpleEditFormType = {
};
dataset: {
datasets: SelectedDatasetType;
searchMode: `${DatasetSearchModeEnum}`;
similarity?: number;
limit?: number;
usingReRank?: boolean;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
};
} & AppDatasetSearchParamsType;
selectedTools: FlowNodeTemplateType[];
chatConfig: AppChatConfigType;
};

View File

@@ -24,9 +24,11 @@ export const getDefaultAppForm = (): AppSimpleEditFormType => {
dataset: {
datasets: [],
similarity: 0.4,
limit: 1500,
limit: 3000,
searchMode: DatasetSearchModeEnum.embedding,
usingReRank: false,
rerankModel: '',
rerankWeight: 0.5,
datasetSearchUsingExtensionQuery: true,
datasetSearchExtensionBg: ''
},
@@ -106,10 +108,24 @@ export const appWorkflow2Form = ({
defaultAppForm.dataset.searchMode =
findInputValueByKey(node.inputs, NodeInputKeyEnum.datasetSearchMode) ||
DatasetSearchModeEnum.embedding;
defaultAppForm.dataset.embeddingWeight = findInputValueByKey(
node.inputs,
NodeInputKeyEnum.datasetSearchEmbeddingWeight
);
// Rerank
defaultAppForm.dataset.usingReRank = !!findInputValueByKey(
node.inputs,
NodeInputKeyEnum.datasetSearchUsingReRank
);
defaultAppForm.dataset.rerankModel = findInputValueByKey(
node.inputs,
NodeInputKeyEnum.datasetSearchRerankModel
);
defaultAppForm.dataset.rerankWeight = findInputValueByKey(
node.inputs,
NodeInputKeyEnum.datasetSearchRerankWeight
);
// Query extension
defaultAppForm.dataset.datasetSearchUsingExtensionQuery = findInputValueByKey(
node.inputs,
NodeInputKeyEnum.datasetSearchUsingExtensionQuery

View File

@@ -185,7 +185,7 @@ export enum SearchScoreTypeEnum {
}
export const SearchScoreTypeMap = {
[SearchScoreTypeEnum.embedding]: {
label: i18nT('common:core.dataset.search.score.embedding'),
label: i18nT('common:core.dataset.search.mode.embedding'),
desc: i18nT('common:core.dataset.search.score.embedding desc'),
showScore: true
},

View File

@@ -154,7 +154,12 @@ export enum NodeInputKeyEnum {
datasetSimilarity = 'similarity',
datasetMaxTokens = 'limit',
datasetSearchMode = 'searchMode',
datasetSearchEmbeddingWeight = 'embeddingWeight',
datasetSearchUsingReRank = 'usingReRank',
datasetSearchRerankWeight = 'rerankWeight',
datasetSearchRerankModel = 'rerankModel',
datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
datasetSearchExtensionModel = 'datasetSearchExtensionModel',
datasetSearchExtensionBg = 'datasetSearchExtensionBg',

View File

@@ -133,6 +133,9 @@ export type DispatchNodeResponseType = {
similarity?: number;
limit?: number;
searchMode?: `${DatasetSearchModeEnum}`;
embeddingWeight?: number;
rerankModel?: string;
rerankWeight?: number;
searchUsingReRank?: boolean;
queryExtensionResult?: {
model: string;

View File

@@ -64,6 +64,14 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
valueType: WorkflowIOValueTypeEnum.string,
value: DatasetSearchModeEnum.embedding
},
{
key: NodeInputKeyEnum.datasetSearchEmbeddingWeight,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.number,
value: 0.5
},
// Rerank
{
key: NodeInputKeyEnum.datasetSearchUsingReRank,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
@@ -71,6 +79,20 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
valueType: WorkflowIOValueTypeEnum.boolean,
value: false
},
{
key: NodeInputKeyEnum.datasetSearchRerankModel,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.string
},
{
key: NodeInputKeyEnum.datasetSearchRerankWeight,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.number,
value: 0.5
},
// Query Extension
{
key: NodeInputKeyEnum.datasetSearchUsingExtensionQuery,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
@@ -91,6 +113,7 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
valueType: WorkflowIOValueTypeEnum.string,
value: ''
},
{
key: NodeInputKeyEnum.authTmbId,
renderTypeList: [FlowNodeInputTypeEnum.hidden],

View File

@@ -8,7 +8,7 @@ import {
EmbeddingModelItemType,
TTSModelType,
STTModelType,
ReRankModelItemType
RerankModelItemType
} from '@fastgpt/global/core/ai/model.d';
import { debounce } from 'lodash';
import {
@@ -94,7 +94,7 @@ export const loadSystemModels = async (init = false) => {
global.embeddingModelMap = new Map<string, EmbeddingModelItemType>();
global.ttsModelMap = new Map<string, TTSModelType>();
global.sttModelMap = new Map<string, STTModelType>();
global.reRankModelMap = new Map<string, ReRankModelItemType>();
global.reRankModelMap = new Map<string, RerankModelItemType>();
// @ts-ignore
global.systemDefaultModel = {};

View File

@@ -38,7 +38,7 @@ export function getSTTModel(model?: string) {
}
export const getDefaultRerankModel = () => global?.systemDefaultModel.rerank!;
export function getReRankModel(model?: string) {
export function getRerankModel(model?: string) {
if (!model) return getDefaultRerankModel();
return global.reRankModelMap.get(model) || getDefaultRerankModel();
}

View File

@@ -2,7 +2,7 @@ import { addLog } from '../../../common/system/log';
import { POST } from '../../../common/api/serverRequest';
import { getDefaultRerankModel } from '../model';
import { getAxiosConfig } from '../config';
import { ReRankModelItemType } from '@fastgpt/global/core/ai/model.d';
import { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
type PostReRankResponse = {
id: string;
@@ -19,7 +19,7 @@ export function reRankRecall({
documents,
headers
}: {
model?: ReRankModelItemType;
model?: RerankModelItemType;
query: string;
documents: { id: string; text: string }[];
headers?: Record<string, string>;

View File

@@ -1,7 +1,7 @@
import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
import {
STTModelType,
ReRankModelItemType,
RerankModelItemType,
TTSModelType,
EmbeddingModelItemType,
LLMModelItemType
@@ -18,7 +18,7 @@ export type SystemModelItemType =
| EmbeddingModelItemType
| TTSModelType
| STTModelType
| ReRankModelItemType;
| RerankModelItemType;
export type SystemDefaultModelType = {
[ModelTypeEnum.llm]?: LLMModelItemType;
@@ -28,7 +28,7 @@ export type SystemDefaultModelType = {
[ModelTypeEnum.embedding]?: EmbeddingModelItemType;
[ModelTypeEnum.tts]?: TTSModelType;
[ModelTypeEnum.stt]?: STTModelType;
[ModelTypeEnum.rerank]?: ReRankModelItemType;
[ModelTypeEnum.rerank]?: RerankModelItemType;
};
declare global {
@@ -38,7 +38,7 @@ declare global {
var embeddingModelMap: Map<string, EmbeddingModelItemType>;
var ttsModelMap: Map<string, TTSModelType>;
var sttModelMap: Map<string, STTModelType>;
var reRankModelMap: Map<string, ReRankModelItemType>;
var reRankModelMap: Map<string, RerankModelItemType>;
var systemActiveModelList: SystemModelItemType[];
var systemDefaultModel: SystemDefaultModelType;

View File

@@ -27,6 +27,7 @@ import { ChatItemType } from '@fastgpt/global/core/chat/type';
import { POST } from '../../../common/api/plusRequest';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { datasetSearchQueryExtension } from './utils';
import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
export type SearchDatasetDataProps = {
histories: ChatItemType[];
@@ -39,7 +40,11 @@ export type SearchDatasetDataProps = {
[NodeInputKeyEnum.datasetSimilarity]?: number; // min distance
[NodeInputKeyEnum.datasetMaxTokens]: number; // max Token limit
[NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
[NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
[NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
/*
{
@@ -75,13 +80,16 @@ export type SearchDatasetDataResponse = {
};
export const datasetDataReRank = async ({
rerankModel,
data,
query
}: {
rerankModel?: RerankModelItemType;
data: SearchDataResponseItemType[];
query: string;
}): Promise<SearchDataResponseItemType[]> => {
const results = await reRankRecall({
model: rerankModel,
query,
documents: data.map((item) => ({
id: item.id,
@@ -154,7 +162,10 @@ export async function searchDatasetData(
similarity = 0,
limit: maxTokens,
searchMode = DatasetSearchModeEnum.embedding,
embeddingWeight = 0.5,
usingReRank = false,
rerankModel,
rerankWeight = 0.5,
datasetIds = [],
collectionFilterMatch
} = props;
@@ -711,6 +722,7 @@ export async function searchDatasetData(
});
try {
return await datasetDataReRank({
rerankModel,
query: reRankQuery,
data: filterSameDataResults
});
@@ -721,11 +733,26 @@ export async function searchDatasetData(
})();
// embedding recall and fullText recall rrf concat
const rrfConcatResults = datasetSearchResultConcat([
{ k: 60, list: embeddingRecallResults },
{ k: 60, list: fullTextRecallResults },
{ k: 58, list: reRankResults }
const baseK = 120;
const embK = Math.round(baseK * (1 - embeddingWeight)); // 搜索结果的 k 值
const fullTextK = Math.round(baseK * embeddingWeight); // rerank 结果的 k 值
const rrfSearchResult = datasetSearchResultConcat([
{ k: embK, list: embeddingRecallResults },
{ k: fullTextK, list: fullTextRecallResults }
]);
const rrfConcatResults = (() => {
if (reRankResults.length === 0) return rrfSearchResult;
if (rerankWeight === 1) return reRankResults;
const searchK = Math.round(baseK * rerankWeight); // 搜索结果的 k 值
const rerankK = Math.round(baseK * (1 - rerankWeight)); // rerank 结果的 k 值
return datasetSearchResultConcat([
{ k: searchK, list: rrfSearchResult },
{ k: rerankK, list: reRankResults }
]);
})();
// remove same q and a data
set = new Set<string>();

View File

@@ -6,7 +6,7 @@ import { formatModelChars2Points } from '../../../../support/wallet/usage/utils'
import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/api.d';
import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
import { getEmbeddingModel } from '../../../ai/model';
import { getEmbeddingModel, getRerankModel } from '../../../ai/model';
import { deepRagSearch, defaultSearchDatasetData } from '../../../dataset/search/controller';
import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
@@ -22,9 +22,14 @@ type DatasetSearchProps = ModuleDispatchProps<{
[NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
[NodeInputKeyEnum.datasetSimilarity]: number;
[NodeInputKeyEnum.datasetMaxTokens]: number;
[NodeInputKeyEnum.datasetSearchMode]: `${DatasetSearchModeEnum}`;
[NodeInputKeyEnum.userChatInput]?: string;
[NodeInputKeyEnum.datasetSearchMode]: `${DatasetSearchModeEnum}`;
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
[NodeInputKeyEnum.datasetSearchUsingReRank]: boolean;
[NodeInputKeyEnum.datasetSearchRerankModel]?: string;
[NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
[NodeInputKeyEnum.collectionFilterMatch]: string;
[NodeInputKeyEnum.authTmbId]?: boolean;
@@ -53,11 +58,14 @@ export async function dispatchDatasetSearch(
datasets = [],
similarity,
limit = 1500,
usingReRank,
searchMode,
userChatInput = '',
authTmbId = false,
collectionFilterMatch,
searchMode,
embeddingWeight,
usingReRank,
rerankModel,
rerankWeight,
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel,
@@ -122,7 +130,10 @@ export async function dispatchDatasetSearch(
limit,
datasetIds,
searchMode,
embeddingWeight,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
rerankModel: getRerankModel(rerankModel),
rerankWeight,
collectionFilterMatch
};
const {
@@ -219,6 +230,9 @@ export async function dispatchDatasetSearch(
similarity: usingSimilarityFilter ? similarity : undefined,
limit,
searchMode,
embeddingWeight: searchMode === DatasetSearchModeEnum.mixedRecall ? embeddingWeight : undefined,
rerankModel: usingReRank ? getRerankModel(rerankModel)?.name : undefined,
rerankWeight: usingReRank ? rerankWeight : undefined,
searchUsingReRank: searchUsingReRank,
quoteList: searchRes,
queryExtensionResult,

View File

@@ -1,7 +1,7 @@
import { FastGPTFeConfigsType, SystemEnvType } from '@fastgpt/global/common/system/types';
import {
TTSModelType,
ReRankModelItemType,
RerankModelItemType,
STTModelType,
EmbeddingModelItemType,
LLMModelItemType

View File

@@ -212,7 +212,9 @@ export const iconPaths = {
'core/dataset/manualCollection': () => import('./icons/core/dataset/manualCollection.svg'),
'core/dataset/mixedRecall': () => import('./icons/core/dataset/mixedRecall.svg'),
'core/dataset/modeEmbedding': () => import('./icons/core/dataset/modeEmbedding.svg'),
'core/dataset/questionExtension': () => import('./icons/core/dataset/questionExtension.svg'),
'core/dataset/rerank': () => import('./icons/core/dataset/rerank.svg'),
'core/dataset/searchfilter': () => import('./icons/core/dataset/searchfilter.svg'),
'core/dataset/splitLight': () => import('./icons/core/dataset/splitLight.svg'),
'core/dataset/tableCollection': () => import('./icons/core/dataset/tableCollection.svg'),
'core/dataset/tag': () => import('./icons/core/dataset/tag.svg'),
@@ -427,6 +429,7 @@ export const iconPaths = {
'price/bg': () => import('./icons/price/bg.svg'),
'price/right': () => import('./icons/price/right.svg'),
save: () => import('./icons/save.svg'),
sliderTag: () => import('./icons/sliderTag.svg'),
stop: () => import('./icons/stop.svg'),
'support/account/laf': () => import('./icons/support/account/laf.svg'),
'support/account/loginoutLight': () => import('./icons/support/account/loginoutLight.svg'),

View File

@@ -0,0 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 17 16" >
<path d="M11.3317 4.63043L12.3128 5.61147L8.98815 8.9361L6.74675 6.6947C6.68372 6.63154 6.60886 6.58144 6.52644 6.54725C6.44402 6.51306 6.35567 6.49546 6.26645 6.49546C6.17722 6.49546 6.08887 6.51306 6.00646 6.54725C5.92404 6.58144 5.84918 6.63154 5.78615 6.6947L1.69849 10.7892C1.63542 10.8522 1.58538 10.9271 1.55125 11.0095C1.51711 11.0919 1.49954 11.1803 1.49954 11.2695C1.49954 11.3587 1.51711 11.447 1.55125 11.5294C1.58538 11.6118 1.63542 11.6867 1.69849 11.7498C1.76156 11.8128 1.83644 11.8629 1.91885 11.897C2.00126 11.9312 2.08959 11.9487 2.17879 11.9487C2.26799 11.9487 2.35632 11.9312 2.43873 11.897C2.52114 11.8629 2.59601 11.8128 2.65909 11.7498L6.26304 8.13901L8.50444 10.3804C8.77014 10.6461 9.19934 10.6461 9.46504 10.3804L13.2734 6.57888L14.2544 7.55992C14.302 7.6066 14.3623 7.63828 14.4277 7.65104C14.4932 7.66379 14.5609 7.65706 14.6226 7.63167C14.6842 7.60628 14.7371 7.56336 14.7746 7.50822C14.8121 7.45308 14.8326 7.38815 14.8335 7.32147V4.39198C14.8354 4.34733 14.8281 4.30276 14.812 4.26104C14.796 4.21931 14.7716 4.18132 14.7403 4.1494C14.709 4.11747 14.6716 4.0923 14.6302 4.07544C14.5888 4.05858 14.5444 4.05038 14.4997 4.05135H11.577C11.5097 4.05095 11.4439 4.07047 11.3877 4.10745C11.3315 4.14443 11.2876 4.1972 11.2613 4.25913C11.2351 4.32106 11.2278 4.38937 11.2403 4.45544C11.2529 4.52152 11.2847 4.5824 11.3317 4.63043Z" />
</svg>

After

Width:  |  Height:  |  Size: 1.4 KiB

View File

@@ -0,0 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 17 16" >
<path fill-rule="evenodd" clip-rule="evenodd" d="M13.8911 3.61032C15.0171 5.17598 14.8761 7.37117 13.4681 8.77915L12.6427 9.60456C12.3733 9.87396 11.9365 9.87396 11.6671 9.60456L6.97652 4.91394C6.70711 4.64454 6.70711 4.20775 6.97652 3.93834L7.80192 3.11294C9.20207 1.71279 11.3807 1.56556 12.9446 2.67127L13.8718 1.74404C14.1322 1.48369 14.5543 1.48369 14.8146 1.74404C15.075 2.00439 15.075 2.4265 14.8146 2.68685L13.8911 3.61032ZM2.18142 13.4425C1.92107 13.7029 1.92107 14.125 2.18142 14.3853C2.44177 14.6457 2.86388 14.6457 3.12423 14.3853L4.04283 13.4667C5.6079 14.5859 7.79751 14.4428 9.20286 13.0374L10.0283 12.212C10.2977 11.9426 10.2977 11.5058 10.0283 11.2364L9.44649 10.6546L9.94334 10.1578C10.2037 9.89744 10.2037 9.47533 9.94334 9.21498C9.68299 8.95463 9.26088 8.95463 9.00053 9.21498L8.50368 9.71183L6.87902 8.08717L7.37233 7.59386C7.63268 7.33351 7.63268 6.9114 7.37233 6.65105C7.11198 6.3907 6.68987 6.3907 6.42952 6.65105L5.93621 7.14436L5.33765 6.5458C5.06824 6.27639 4.63145 6.27639 4.36205 6.5458L3.53665 7.3712C2.13385 8.774 1.98872 10.9582 3.10124 12.5227L2.18142 13.4425ZM4.84985 7.94362L4.47946 8.31401C3.43548 9.35799 3.43548 11.0506 4.47946 12.0946C5.52344 13.1386 7.21607 13.1386 8.26005 12.0946L8.63044 11.7242L4.84985 7.94362ZM12.5253 7.83635L12.1549 8.20674L8.37433 4.42614L8.74473 4.05575C9.78871 3.01177 11.4813 3.01177 12.5253 4.05575C13.5693 5.09973 13.5693 6.79236 12.5253 7.83635Z" />
</svg>

After

Width:  |  Height:  |  Size: 1.5 KiB

View File

@@ -0,0 +1,3 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 11 11" >
<path d="M5.04123 0.144501L9.47821 4.82132C9.83075 5.19292 10.0273 5.68562 10.0273 6.19784V8.65565C10.0273 9.76022 9.13185 10.6557 8.02728 10.6557H2.05518C0.950606 10.6557 0.0551758 9.76022 0.0551758 8.65565V6.19785C0.0551758 5.68562 0.251705 5.19292 0.604247 4.82132L5.04123 0.144501Z" fill="#3370FF"/>
</svg>

After

Width:  |  Height:  |  Size: 375 B

View File

@@ -53,6 +53,7 @@ const MyModal = ({
allowPinchZoom
scrollBehavior={'inside'}
closeOnOverlayClick={closeOnOverlayClick}
returnFocusOnClose={false}
>
<ModalOverlay />
<ModalContent

View File

@@ -73,6 +73,11 @@ const LightRowTabs = <ValueType = string,>({
color: activeColor
}}
fontWeight={'medium'}
onClick={() => {
if (value === item.value) return;
onChange(item.value);
}}
{...inlineStyles}
{...(value === item.value
? {
color: activeColor,
@@ -82,11 +87,6 @@ const LightRowTabs = <ValueType = string,>({
: {
cursor: 'pointer'
})}
onClick={() => {
if (value === item.value) return;
onChange(item.value);
}}
{...inlineStyles}
>
{item.icon && (
<>

View File

@@ -1,4 +1,4 @@
import { useEffect, useRef, useState, ReactNode, useCallback } from 'react';
import { useEffect, useRef, useState, ReactNode } from 'react';
import { LinkedListResponse, LinkedPaginationProps } from '../common/fetch/type';
import { Box, BoxProps } from '@chakra-ui/react';
import { useTranslation } from 'next-i18next';
@@ -34,13 +34,13 @@ export function useLinkedScroll<
bottom: null as { _id: string; index: number } | null
});
const containerRef = useRef<HTMLDivElement>(null);
const itemRefs = useRef<(HTMLElement | null)[]>([]);
const itemRefs = useRef<Map<string, HTMLElement | null>>(new Map());
const scrollToItem = async (id: string, retry = 3) => {
const itemIndex = dataList.findIndex((item) => item._id === id);
if (itemIndex === -1) return;
const element = itemRefs.current[itemIndex];
const element = itemRefs.current.get(id);
if (!element || !containerRef.current) {
if (retry > 0) {
@@ -64,11 +64,12 @@ export function useLinkedScroll<
let scroolSign = useRef(false);
const { runAsync: loadInitData } = useRequest2(
async (scrollWhenFinish = true) => {
async ({ scrollWhenFinish, refresh } = { scrollWhenFinish: true, refresh: false }) => {
console.log('loadInitData', params);
if (!currentData || isLoading) return;
const item = dataList.find((item) => item._id === currentData.id);
if (item) {
if (item && !refresh) {
scrollToItem(item._id);
return;
}

View File

@@ -49,6 +49,7 @@
"response.child total points": "Sub-workflow point consumption",
"response.dataset_concat_length": "Combined total",
"response.node_inputs": "Node Inputs",
"response_hybrid_weight": "Embedding : Full text = {{emb}} : {{text}}",
"select": "Select",
"select_file": "Upload File",
"select_file_img": "Upload file / image",

View File

@@ -107,6 +107,7 @@
"code_error.user_error.balance_not_enough": "Insufficient Account Balance",
"code_error.user_error.bin_visitor_guest": "You Are Currently a Guest, Unauthorized to Operate",
"code_error.user_error.un_auth_user": "User Not Found",
"commercial_function_tip": "Please Upgrade to the Commercial Version to Use This Feature: https://doc.fastgpt.cn/docs/commercial/intro/",
"common.Action": "Action",
"common.Add": "Add",
"common.Add New": "Add New",
@@ -256,7 +257,6 @@
"common.submit_success": "Submitted Successfully",
"common.submitted": "Submitted",
"common.support": "Support",
"commercial_function_tip": "Please Upgrade to the Commercial Version to Use This Feature: https://doc.fastgpt.cn/docs/commercial/intro/",
"common.system.Help Chatbot": "Help Chatbot",
"common.system.Use Helper": "Use Helper",
"common.ui.textarea.Magnifying": "Magnifying",
@@ -627,7 +627,7 @@
"core.dataset.search.score.reRank desc": "Calculate the relevance between sentences using the re-rank model, ranging from 0 to 1.",
"core.dataset.search.score.rrf": "Comprehensive Ranking",
"core.dataset.search.score.rrf desc": "Merge multiple search results using the reciprocal rank fusion method.",
"core.dataset.search.search mode": "Search Mode",
"core.dataset.search.search mode": "Search Method",
"core.dataset.status.active": "Ready",
"core.dataset.status.syncing": "Syncing",
"core.dataset.test.Batch test": "Batch Test",
@@ -1025,6 +1025,7 @@
"question_feedback": "Work order",
"read_quote": "View citations",
"required": "Required",
"rerank_weight": "Rearrange weights",
"resume_failed": "Resume Failed",
"select_reference_variable": "Select Reference Variable",
"share_link": "Share Link",

View File

@@ -49,6 +49,7 @@
"response.child total points": "子工作流积分消耗",
"response.dataset_concat_length": "合并后总数",
"response.node_inputs": "节点输入",
"response_hybrid_weight": "语义检索 : 全文检索 = {{emb}} : {{text}}",
"select": "选择",
"select_file": "上传文件",
"select_file_img": "上传文件/图片",

View File

@@ -111,6 +111,7 @@
"code_error.user_error.balance_not_enough": "账号余额不足~",
"code_error.user_error.bin_visitor_guest": "您当前身份为游客,无权操作",
"code_error.user_error.un_auth_user": "找不到该用户",
"commercial_function_tip": "请升级商业版后使用该功能https://doc.fastgpt.cn/docs/commercial/intro/",
"common.Action": "操作",
"common.Add": "添加",
"common.Add New": "新增",
@@ -260,7 +261,6 @@
"common.submit_success": "提交成功",
"common.submitted": "已提交",
"common.support": "支持",
"commercial_function_tip": "请升级商业版后使用该功能https://doc.fastgpt.cn/docs/commercial/intro/",
"common.system.Help Chatbot": "机器人助手",
"common.system.Use Helper": "使用帮助",
"common.ui.textarea.Magnifying": "放大",
@@ -623,7 +623,6 @@
"core.dataset.search.mode.fullTextRecall desc": "使用传统的全文检索,适合查找一些关键词和主谓语特殊的数据",
"core.dataset.search.mode.mixedRecall": "混合检索",
"core.dataset.search.mode.mixedRecall desc": "使用向量检索与全文检索的综合结果返回,使用 RRF 算法进行排序。",
"core.dataset.search.score.embedding": "语义检索",
"core.dataset.search.score.embedding desc": "通过计算向量之间的距离获取得分,范围为 0~1。",
"core.dataset.search.score.fullText": "全文检索",
"core.dataset.search.score.fullText desc": "计算相同关键词的得分,范围为 0~无穷。",
@@ -631,7 +630,7 @@
"core.dataset.search.score.reRank desc": "通过 Rerank 模型计算句子之间的关联度,范围为 0~1。",
"core.dataset.search.score.rrf": "综合排名",
"core.dataset.search.score.rrf desc": "通过倒排计算的方式,合并多个检索结果。",
"core.dataset.search.search mode": "搜索式",
"core.dataset.search.search mode": "搜索式",
"core.dataset.status.active": "已就绪",
"core.dataset.status.syncing": "同步中",
"core.dataset.test.Batch test": "批量测试",
@@ -1029,6 +1028,7 @@
"question_feedback": "工单咨询",
"read_quote": "查看引用",
"required": "必须",
"rerank_weight": "重排权重",
"resume_failed": "恢复失败",
"select_reference_variable": "选择引用变量",
"share_link": "分享链接",

View File

@@ -48,6 +48,7 @@
"response.child total points": "子工作流程點數消耗",
"response.dataset_concat_length": "合併總數",
"response.node_inputs": "節點輸入",
"response_hybrid_weight": "語義檢索 : 全文檢索 = {{emb}} : {{text}}",
"select": "選取",
"select_file": "上傳檔案",
"select_file_img": "上傳檔案 / 圖片",

View File

@@ -106,6 +106,7 @@
"code_error.user_error.balance_not_enough": "帳戶餘額不足",
"code_error.user_error.bin_visitor_guest": "您目前身份為訪客,無權操作",
"code_error.user_error.un_auth_user": "找不到此使用者",
"commercial_function_tip": "請升級為商業版後使用此功能https://doc.fastgpt.cn/docs/commercial/intro/",
"common.Action": "操作",
"common.Add": "新增",
"common.Add New": "新增",
@@ -255,7 +256,6 @@
"common.submit_success": "送出成功",
"common.submitted": "已送出",
"common.support": "支援",
"commercial_function_tip": "請升級為商業版後使用此功能https://doc.fastgpt.cn/docs/commercial/intro/",
"common.system.Help Chatbot": "機器人助手",
"common.system.Use Helper": "使用說明",
"common.ui.textarea.Magnifying": "放大",
@@ -626,7 +626,7 @@
"core.dataset.search.score.reRank desc": "透過重新排名模型計算句子之間的關聯度,範圍為 0 到 1。",
"core.dataset.search.score.rrf": "綜合排名",
"core.dataset.search.score.rrf desc": "使用倒數排名融合方法,合併多個搜尋結果。",
"core.dataset.search.search mode": "搜尋模式",
"core.dataset.search.search mode": "搜索方式",
"core.dataset.status.active": "已就緒",
"core.dataset.status.syncing": "同步中",
"core.dataset.test.Batch test": "批次測試",
@@ -1024,6 +1024,7 @@
"question_feedback": "工單諮詢",
"read_quote": "查看引用",
"required": "必填",
"rerank_weight": "重排權重",
"resume_failed": "恢復失敗",
"select_reference_variable": "選擇引用變數",
"share_link": "分享連結",

View File

@@ -1 +0,0 @@
<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1703041511149" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="7024" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M512.512 128a384 384 0 1 1-384 384 384 384 0 0 1 384-384z m0 96a288 288 0 1 0 288 288 288 288 0 0 0-288-288z" fill="#3671FD" opacity=".2" p-id="7025"></path><path d="M512.512 64a448 448 0 1 1-302.56 117.6A32.032 32.032 0 1 1 253.312 228.8a384 384 0 1 0 298.016-98.88l-6.816-0.608v140.032a32 32 0 0 1-28.256 31.776l-3.744 0.224a32 32 0 0 1-31.776-28.288l-0.224-3.712V64z" fill="#3671FD" p-id="7026"></path><path d="M336.32 313.376l216.224 165.344a52.288 52.288 0 1 1-73.28 73.312l-165.376-216.224a16 16 0 0 1 22.4-22.4z" fill="#FE9C23" p-id="7027"></path></svg>

Before

Width:  |  Height:  |  Size: 893 B

View File

@@ -85,7 +85,7 @@ export default React.memo(Markdown);
function Code(e: any) {
const { className, codeBlock, children } = e;
const match = /language-(\w+)/.exec(className || '');
const codeType = match?.[1];
const codeType = match?.[1]?.toLowerCase();
const strChildren = String(children);
@@ -105,7 +105,7 @@ function Code(e: any) {
if (codeType === CodeClassNameEnum.iframe) {
return <IframeCodeBlock code={strChildren} />;
}
if (codeType && codeType.toLowerCase() === CodeClassNameEnum.html) {
if (codeType === CodeClassNameEnum.html || codeType === CodeClassNameEnum.svg) {
return (
<IframeHtmlCodeBlock className={className} codeBlock={codeBlock} match={match}>
{children}

View File

@@ -8,6 +8,7 @@ export enum CodeClassNameEnum {
latex = 'latex',
iframe = 'iframe',
html = 'html',
svg = 'svg',
video = 'video',
audio = 'audio'
}

View File

@@ -2,13 +2,15 @@ import React, { useEffect, useMemo, useState } from 'react';
import {
Box,
Button,
Checkbox,
Divider,
Flex,
HStack,
ModalBody,
ModalFooter,
Switch,
useTheme
Slider,
SliderTrack,
SliderFilledTrack,
SliderThumb
} from '@chakra-ui/react';
import { useForm } from 'react-hook-form';
import MyModal from '@fastgpt/web/components/common/MyModal';
@@ -17,30 +19,18 @@ import { useTranslation } from 'next-i18next';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constants';
import MyRadio from '@/components/common/MyRadio';
import MyIcon from '@fastgpt/web/components/common/Icon';
import LightRowTabs from '@fastgpt/web/components/common/Tabs/LightRowTabs';
import { useUserStore } from '@/web/support/user/useUserStore';
import { useToast } from '@fastgpt/web/hooks/useToast';
import SelectAiModel from '@/components/Select/AIModelSelector';
import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
import MyTextarea from '@/components/common/Textarea/MyTextarea';
import { defaultDatasetMaxTokens } from '@fastgpt/global/core/app/constants';
import InputSlider from '@fastgpt/web/components/common/MySlider/InputSlider';
import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
import { AppDatasetSearchParamsType } from '@fastgpt/global/core/app/type';
import MyIcon from '@fastgpt/web/components/common/Icon';
export type DatasetParamsProps = {
searchMode: `${DatasetSearchModeEnum}`;
limit?: number;
similarity?: number;
usingReRank?: boolean;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
maxTokens?: number; // limit max tokens
};
enum SearchSettingTabEnum {
searchMode = 'searchMode',
limit = 'limit',
@@ -51,17 +41,22 @@ const DatasetParamsModal = ({
searchMode = DatasetSearchModeEnum.embedding,
limit,
similarity,
embeddingWeight,
usingReRank,
maxTokens = defaultDatasetMaxTokens,
rerankModel,
rerankWeight,
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel,
datasetSearchExtensionBg,
maxTokens = defaultDatasetMaxTokens,
onClose,
onSuccess
}: DatasetParamsProps & { onClose: () => void; onSuccess: (e: DatasetParamsProps) => void }) => {
}: AppDatasetSearchParamsType & {
maxTokens?: number; // limit max tokens
onClose: () => void;
onSuccess: (e: AppDatasetSearchParamsType) => void;
}) => {
const { t } = useTranslation();
const theme = useTheme();
const { toast } = useToast();
const { teamPlanStatus } = useUserStore();
const { reRankModelList, llmModelList, defaultModels } = useSystemStore();
const [refresh, setRefresh] = useState(false);
@@ -72,28 +67,41 @@ const DatasetParamsModal = ({
value: item.model,
label: item.name
})))();
const reRankModelSelectList = (() =>
reRankModelList.map((item) => ({
value: item.model,
label: item.name
})))();
const { register, setValue, getValues, handleSubmit, watch } =
useForm<AppDatasetSearchParamsType>({
defaultValues: {
searchMode,
embeddingWeight: embeddingWeight || 0.5,
usingReRank: !!usingReRank && teamPlanStatus?.standardConstants?.permissionReRank !== false,
rerankModel: rerankModel || defaultModels?.rerank?.model,
rerankWeight: rerankWeight || 0.5,
limit,
similarity,
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel: datasetSearchExtensionModel || defaultModels.llm?.model,
datasetSearchExtensionBg
}
});
const searchModeWatch = watch('searchMode');
const embeddingWeightWatch = watch('embeddingWeight');
const fullTextWeightWatch = useMemo(() => {
const val = 1 - (embeddingWeightWatch || 0.5);
return Number(val.toFixed(2));
}, [embeddingWeightWatch]);
const { register, setValue, getValues, handleSubmit, watch } = useForm<DatasetParamsProps>({
defaultValues: {
limit,
similarity,
searchMode,
usingReRank: !!usingReRank && teamPlanStatus?.standardConstants?.permissionReRank !== false,
datasetSearchUsingExtensionQuery,
datasetSearchExtensionModel: datasetSearchExtensionModel || defaultModels.llm?.model,
datasetSearchExtensionBg
}
});
const datasetSearchUsingCfrForm = watch('datasetSearchUsingExtensionQuery');
const queryExtensionModel = watch('datasetSearchExtensionModel');
const cfbBgDesc = watch('datasetSearchExtensionBg');
const usingReRankWatch = watch('usingReRank');
const searchModeWatch = watch('searchMode');
const searchModeList = useMemo(() => {
const list = Object.values(DatasetSearchModeMap);
return list;
}, []);
const usingReRankWatch = watch('usingReRank');
const reRankModelWatch = watch('rerankModel');
const rerankWeightWatch = watch('rerankWeight');
const showSimilarity = useMemo(() => {
if (similarity === undefined) return false;
@@ -134,93 +142,160 @@ const DatasetParamsModal = ({
title={t('common:core.dataset.search.Dataset Search Params')}
w={['90vw', '550px']}
>
<ModalBody flex={'auto'} overflow={'auto'}>
<ModalBody flex={'auto'} overflow={'auto'} px={[4, 10]}>
<LightRowTabs<SearchSettingTabEnum>
width={'100%'}
mb={3}
list={[
{
icon: 'modal/setting',
icon: 'common/setting',
label: t('common:core.dataset.search.search mode'),
value: SearchSettingTabEnum.searchMode
},
{
icon: 'support/outlink/apikeyFill',
icon: 'core/dataset/searchfilter',
label: t('common:core.dataset.search.Filter'),
value: SearchSettingTabEnum.limit
},
{
label: t('common:core.module.template.Query extension'),
value: SearchSettingTabEnum.queryExtension,
icon: '/imgs/workflow/cfr.svg'
icon: 'core/dataset/questionExtension'
}
]}
inlineStyles={{
borderBottomColor: 'myGray.200',
borderBottom: '1px solid'
}}
value={currentTabType}
onChange={setCurrentTabType}
/>
{currentTabType === SearchSettingTabEnum.searchMode && (
<>
<MyRadio
gridGap={2}
gridTemplateColumns={'repeat(1,1fr)'}
list={searchModeList}
value={getValues('searchMode')}
<Box mt={3}>
<LeftRadio<`${DatasetSearchModeEnum}`>
py={2.5}
gridGap={4}
list={[
{
title: t('common:core.dataset.search.mode.embedding'),
desc: t('common:core.dataset.search.mode.embedding desc'),
value: DatasetSearchModeEnum.embedding
},
{
title: t('common:core.dataset.search.mode.fullTextRecall'),
desc: t('common:core.dataset.search.mode.fullTextRecall desc'),
value: DatasetSearchModeEnum.fullTextRecall
},
{
title: t('common:core.dataset.search.mode.mixedRecall'),
desc: t('common:core.dataset.search.mode.mixedRecall desc'),
value: DatasetSearchModeEnum.mixedRecall,
children: searchModeWatch === DatasetSearchModeEnum.mixedRecall && (
<Box mt={3}>
<HStack justifyContent={'space-between'}>
<Flex alignItems={'center'}>
<Box fontSize={'sm'} color={'myGray.900'}>
{t('common:core.dataset.search.mode.embedding')}
</Box>
<Box fontSize={'xs'} color={'myGray.500'}>
{embeddingWeightWatch}
</Box>
</Flex>
<Flex alignItems={'center'}>
<Box fontSize={'sm'} color={'myGray.900'}>
{t('common:core.dataset.search.score.fullText')}
</Box>
<Box fontSize={'xs'} color={'myGray.500'}>
{fullTextWeightWatch}
</Box>
</Flex>
</HStack>
<Slider
defaultValue={embeddingWeightWatch}
min={0.1}
max={0.9}
step={0.01}
onChange={(e) => {
setValue('embeddingWeight', Number(e.toFixed(2)));
}}
>
<SliderTrack bg={'#F9518E'}>
<SliderFilledTrack bg={'#3370FF'} />
</SliderTrack>
<SliderThumb boxShadow={'none'} bg={'none'}>
<MyIcon transform={'translateY(10px)'} name={'sliderTag'} w={'1rem'} />
</SliderThumb>
</Slider>
</Box>
)
}
]}
value={searchModeWatch}
onChange={(e) => {
setValue('searchMode', e as `${DatasetSearchModeEnum}`);
setRefresh(!refresh);
setValue('searchMode', e);
}}
/>
{/* Rerank */}
<>
<Divider my={4} />
<Flex
alignItems={'center'}
cursor={'pointer'}
userSelect={'none'}
py={3}
px={4}
border={theme.borders.sm}
borderWidth={'1.5px'}
borderRadius={'md'}
position={'relative'}
{...(getValues('usingReRank')
? {
borderColor: 'primary.400'
}
: {})}
onClick={(e) => {
if (!showReRank) {
return toast({
status: 'warning',
title: t('common:core.ai.Not deploy rerank model')
});
}
if (
teamPlanStatus?.standardConstants &&
!teamPlanStatus?.standardConstants?.permissionReRank
) {
return toast({
status: 'warning',
title: t('common:support.team.limit.No permission rerank')
});
}
setValue('usingReRank', !getValues('usingReRank'));
setRefresh((state) => !state);
}}
>
<MyIcon name="core/dataset/rerank" w={'18px'} mr={'14px'} />
<Box pr={2} color={'myGray.800'} flex={'1 0 0'}>
<Box fontSize={'sm'}>{t('common:core.dataset.search.ReRank')}</Box>
<Box fontSize={'xs'} color={'myGray.500'}>
{t('common:core.dataset.search.ReRank desc')}
<HStack mt={6} justifyContent={'space-between'}>
<FormLabel>
{t('common:core.dataset.search.ReRank')}
<QuestionTip ml={0.5} label={t('common:core.dataset.search.ReRank desc')} />
</FormLabel>
{!showReRank ? (
<Box color={'myGray.500'} fontSize={'sm'}>
{t('common:core.ai.Not deploy rerank model')}
</Box>
</Box>
<Box position={'relative'} w={'18px'} h={'18px'}>
<Checkbox colorScheme="primary" isChecked={getValues('usingReRank')} size="lg" />
<Box position={'absolute'} top={0} right={0} bottom={0} left={0} zIndex={1}></Box>
</Box>
</Flex>
) : teamPlanStatus?.standardConstants &&
!teamPlanStatus?.standardConstants?.permissionReRank ? (
<Box color={'myGray.500'} fontSize={'sm'}>
{t('common:support.team.limit.No permission rerank')}
</Box>
) : (
<Switch {...register('usingReRank')} />
)}
</HStack>
{usingReRankWatch && (
<>
<HStack mt={3} justifyContent={'space-between'}>
<Box fontSize={'sm'} flex={'0 0 100px'} color={'myGray.700'}>
{t('common:rerank_weight')}
</Box>
<Box flex={'1 0 0'}>
<InputSlider
min={0.1}
max={1}
step={0.01}
value={rerankWeightWatch}
onChange={(val) => {
setValue(
NodeInputKeyEnum.datasetSearchRerankWeight,
Number(val.toFixed(2))
);
}}
/>
</Box>
</HStack>
<HStack mt={3}>
<Box fontSize={'sm'} flex={'0 0 100px'} color={'myGray.700'}>
{t('common:model.type.reRank')}
</Box>
<Box flex={'1 0 0'}>
<SelectAiModel
bg={'myGray.50'}
h={'36px'}
value={reRankModelWatch}
list={reRankModelSelectList}
onChange={(val) => {
setValue(NodeInputKeyEnum.datasetSearchRerankModel, val);
}}
/>
</Box>
</HStack>
</>
)}
</>
</>
</Box>
)}
{currentTabType === SearchSettingTabEnum.limit && (
<Box pt={5}>
@@ -262,7 +337,7 @@ const DatasetParamsModal = ({
}}
/>
) : (
<Box color={'myGray.500'}>
<Box color={'myGray.500'} fontSize={'sm'}>
{t('common:core.dataset.search.No support similarity')}
</Box>
)}

View File

@@ -299,6 +299,7 @@ const RenderUserFormInteractive = React.memo(function RenderFormInput({
<MyNumberInput
min={input.min}
max={input.max}
defaultValue={input.defaultValue}
isDisabled={interactive.params.submitted}
bg={'white'}
register={register}

View File

@@ -228,10 +228,23 @@ export const WholeResponseContent = ({
{activeModule?.searchMode && (
<Row
label={t('common:core.dataset.search.search mode')}
// @ts-ignore
value={t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
rawDom={
<Flex border={'base'} borderRadius={'md'} p={2}>
<Box>
{/* @ts-ignore */}
{t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
</Box>
{activeModule.embeddingWeight && (
<>{`(${t('chat:response_hybrid_weight', {
emb: activeModule.embeddingWeight,
text: 1 - activeModule.embeddingWeight
})})`}</>
)}
</Flex>
}
/>
)}
<Row
label={t('common:core.chat.response.module similarity')}
value={activeModule?.similarity}
@@ -239,7 +252,19 @@ export const WholeResponseContent = ({
<Row label={t('common:core.chat.response.module limit')} value={activeModule?.limit} />
<Row
label={t('common:core.chat.response.search using reRank')}
value={`${activeModule?.searchUsingReRank}`}
rawDom={
<Box border={'base'} borderRadius={'md'} p={2}>
{activeModule?.searchUsingReRank ? (
activeModule?.rerankModel ? (
<Box>{`${activeModule.rerankModel}: ${activeModule.rerankWeight}`}</Box>
) : (
'True'
)
) : (
`False`
)}
</Box>
}
/>
{activeModule.queryExtensionResult && (
<>

View File

@@ -3,7 +3,7 @@ import type {
EmbeddingModelItemType,
AudioSpeechModels,
STTModelType,
ReRankModelItemType
RerankModelItemType
} from '@fastgpt/global/core/ai/model.d';
import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/index.d';

View File

@@ -63,8 +63,13 @@ export type SearchTestProps = {
text: string;
[NodeInputKeyEnum.datasetSimilarity]?: number;
[NodeInputKeyEnum.datasetMaxTokens]?: number;
[NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
[NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
[NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
[NodeInputKeyEnum.datasetSearchRerankModel]?: string;
[NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
[NodeInputKeyEnum.datasetSearchUsingExtensionQuery]?: boolean;
[NodeInputKeyEnum.datasetSearchExtensionModel]?: string;

View File

@@ -280,6 +280,10 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
isCustom: true,
isActive: true,
isDefault: false,
isDefaultDatasetTextModel: false,
isDefaultDatasetImageModel: false,
// @ts-ignore
type
});

View File

@@ -10,7 +10,6 @@ import {
HStack
} from '@chakra-ui/react';
import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d';
import type { DatasetSimpleItemType } from '@fastgpt/global/core/dataset/type.d';
import { useRouter } from 'next/router';
import { useTranslation } from 'next-i18next';
@@ -106,6 +105,7 @@ const EditForm = ({
const tokenLimit = useMemo(() => {
return selectedModel?.quoteMaxToken || 3000;
}, [selectedModel?.quoteMaxToken]);
// Force close image select when model not support vision
useEffect(() => {
if (!selectedModel.vision) {
@@ -434,8 +434,6 @@ const EditForm = ({
...e
}
}));
console.dir(e);
}}
/>
)}

View File

@@ -28,8 +28,6 @@ const SimpleEdit = () => {
// Init app form
useMount(() => {
// show selected dataset
if (appDetail.version !== 'v2') {
return setAppForm(
appWorkflow2Form({

View File

@@ -6,13 +6,14 @@ import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import MyIcon from '@fastgpt/web/components/common/Icon';
import DatasetParamsModal, { DatasetParamsProps } from '@/components/core/app/DatasetParamsModal';
import DatasetParamsModal from '@/components/core/app/DatasetParamsModal';
import { useSystemStore } from '@/web/common/system/useSystemStore';
import SearchParamsTip from '@/components/core/dataset/SearchParamsTip';
import { useContextSelector } from 'use-context-selector';
import { WorkflowContext } from '@/pageComponents/app/detail/WorkflowComponents/context';
import { getWebLLMModel } from '@/web/common/system/utils';
import { defaultDatasetMaxTokens } from '@fastgpt/global/core/app/constants';
import { AppDatasetSearchParamsType } from '@fastgpt/global/core/app/type';
const SelectDatasetParam = ({ inputs = [], nodeId }: RenderInputProps) => {
const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode);
@@ -21,11 +22,14 @@ const SelectDatasetParam = ({ inputs = [], nodeId }: RenderInputProps) => {
const { t } = useTranslation();
const { defaultModels } = useSystemStore();
const [data, setData] = useState<DatasetParamsProps>({
const [data, setData] = useState<AppDatasetSearchParamsType>({
searchMode: DatasetSearchModeEnum.embedding,
limit: 5,
embeddingWeight: 0.5,
limit: 3000,
similarity: 0.5,
usingReRank: false,
rerankModel: defaultModels.llm?.model,
rerankWeight: 0.6,
datasetSearchUsingExtensionQuery: true,
datasetSearchExtensionModel: defaultModels.llm?.model,
datasetSearchExtensionBg: ''

View File

@@ -8,7 +8,6 @@ import { useCopyData } from '@fastgpt/web/hooks/useCopyData';
import InputDataModal from '@/pageComponents/dataset/detail/InputDataModal';
const CollectionQuoteItem = ({
index,
quoteRefs,
quoteIndex,
setQuoteIndex,
@@ -22,8 +21,7 @@ const CollectionQuoteItem = ({
dataId,
collectionId
}: {
index: number;
quoteRefs: MutableRefObject<(HTMLDivElement | null)[]>;
quoteRefs: MutableRefObject<Map<string, HTMLDivElement | null>>;
quoteIndex: number;
setQuoteIndex: Dispatch<SetStateAction<number>>;
refreshList: () => void;
@@ -45,7 +43,7 @@ const CollectionQuoteItem = ({
<>
<Box
ref={(el: HTMLDivElement | null) => {
quoteRefs.current[index] = el;
quoteRefs.current.set(dataId, el);
}}
p={2}
py={2}
@@ -163,9 +161,7 @@ const CollectionQuoteItem = ({
'0px 1px 2px 0px rgba(19, 51, 107, 0.05), 0px 0px 1px 0px rgba(19, 51, 107, 0.08)'
}
cursor={'pointer'}
onClick={() => {
copyData(q + '\n' + a);
}}
onClick={() => copyData(`${q}${a ? '\n' + a : ''}`)}
>
<MyIcon name="copy" w={'14px'} color={'myGray.500'} />
</Flex>

View File

@@ -256,11 +256,10 @@ const CollectionReader = ({
{formatedDataList.map((item, index) => (
<CollectionQuoteItem
key={item._id}
index={index}
quoteRefs={itemRefs as React.MutableRefObject<(HTMLDivElement | null)[]>}
quoteRefs={itemRefs as React.MutableRefObject<Map<string, HTMLDivElement | null>>}
quoteIndex={item.quoteIndex}
setQuoteIndex={setQuoteIndex}
refreshList={() => loadInitData(false)}
refreshList={() => loadInitData({ scrollWhenFinish: false, refresh: true })}
updated={item.updated}
isCurrentSelected={item.isCurrentSelected}
q={item.q}

View File

@@ -60,7 +60,7 @@ const ApiDatasetForm = ({
<Input
bg={'myWhite.600'}
placeholder={t('dataset:request_headers')}
maxLength={200}
maxLength={2000}
{...register('apiServer.authorization')}
/>
</Flex>

View File

@@ -36,9 +36,14 @@ type FormType = {
inputText: string;
searchParams: {
searchMode: `${DatasetSearchModeEnum}`;
embeddingWeight?: number;
usingReRank?: boolean;
rerankModel?: string;
rerankWeight?: number;
similarity?: number;
limit?: number;
usingReRank?: boolean;
datasetSearchUsingExtensionQuery?: boolean;
datasetSearchExtensionModel?: string;
datasetSearchExtensionBg?: string;
@@ -53,7 +58,6 @@ const Test = ({ datasetId }: { datasetId: string }) => {
const { pushDatasetTestItem } = useSearchTestStore();
const [inputType, setInputType] = useState<'text' | 'file'>('text');
const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
const [refresh, setRefresh] = useState(false);
const [isFocus, setIsFocus] = useState(false);
const { File, onOpen } = useSelectFile({
fileType: '.csv',
@@ -66,7 +70,10 @@ const Test = ({ datasetId }: { datasetId: string }) => {
inputText: '',
searchParams: {
searchMode: DatasetSearchModeEnum.embedding,
embeddingWeight: 0.5,
usingReRank: false,
rerankModel: defaultModels?.rerank?.model,
rerankWeight: 0.5,
limit: 5000,
similarity: 0,
datasetSearchUsingExtensionQuery: false,
@@ -77,6 +84,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
});
const searchModeData = DatasetSearchModeMap[getValues(`searchParams.searchMode`)];
const searchParams = getValues('searchParams');
const {
isOpen: isOpenSelectMode,
@@ -294,15 +302,14 @@ const Test = ({ datasetId }: { datasetId: string }) => {
{isOpenSelectMode && (
<DatasetParamsModal
{...getValues('searchParams')}
{...searchParams}
maxTokens={20000}
onClose={onCloseSelectMode}
onSuccess={(e) => {
setValue('searchParams', {
...getValues('searchParams'),
...searchParams,
...e
});
setRefresh((state) => !state);
}}
/>
)}

View File

@@ -5,7 +5,7 @@ import { findModelFromAlldata } from '@fastgpt/service/core/ai/model';
import {
EmbeddingModelItemType,
LLMModelItemType,
ReRankModelItemType,
RerankModelItemType,
STTModelType,
TTSModelType
} from '@fastgpt/global/core/ai/model.d';
@@ -151,7 +151,7 @@ const testSTTModel = async (model: STTModelType, headers: Record<string, string>
addLog.info(`STT result: ${text}`);
};
const testReRankModel = async (model: ReRankModelItemType, headers: Record<string, string>) => {
const testReRankModel = async (model: RerankModelItemType, headers: Record<string, string>) => {
await reRankRecall({
model,
query: 'Hi',

View File

@@ -27,13 +27,6 @@ async function handler(
const dbModel = await MongoSystemModel.findOne({ model }).lean();
const modelData = findModelFromAlldata(model);
if (metadata) {
delete metadata.isActive;
delete metadata.isDefault;
delete metadata.isDefaultDatasetTextModel;
delete metadata.isDefaultDatasetImageModel;
}
const metadataConcat: Record<string, any> = {
...modelData, // system config
...dbModel?.metadata, // db config

View File

@@ -16,6 +16,7 @@ import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
import { ApiRequestProps } from '@fastgpt/service/type/next';
import { getRerankModel } from '@fastgpt/service/core/ai/model';
async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTestResponse> {
const {
@@ -24,7 +25,11 @@ async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTes
limit = 1500,
similarity,
searchMode,
embeddingWeight,
usingReRank,
rerankModel,
rerankWeight,
datasetSearchUsingExtensionQuery = false,
datasetSearchExtensionModel,
@@ -63,7 +68,10 @@ async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTes
similarity,
datasetIds: [datasetId],
searchMode,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId))
embeddingWeight,
usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
rerankModel: getRerankModel(rerankModel),
rerankWeight
};
const { searchRes, tokens, queryExtensionResult, deepSearchResult, ...result } = datasetDeepSearch
? await deepRagSearch({

View File

@@ -3,7 +3,7 @@ import {
ChatModelItemType,
FunctionModelItemType,
LLMModelItemType,
ReRankModelItemType,
RerankModelItemType,
EmbeddingModelItemType,
STTModelType
} from '@fastgpt/global/core/ai/model.d';

View File

@@ -6,7 +6,7 @@ import { OAuthEnum } from '@fastgpt/global/support/user/constant';
import type {
TTSModelType,
LLMModelItemType,
ReRankModelItemType,
RerankModelItemType,
EmbeddingModelItemType,
STTModelType
} from '@fastgpt/global/core/ai/model.d';
@@ -56,7 +56,7 @@ type State = {
getVlmModelList: () => LLMModelItemType[];
embeddingModelList: EmbeddingModelItemType[];
ttsModelList: TTSModelType[];
reRankModelList: ReRankModelItemType[];
reRankModelList: RerankModelItemType[];
sttModelList: STTModelType[];
initStaticData: (e: InitDateResponse) => void;
appType?: string;

View File

@@ -239,58 +239,79 @@ export function form2AppWorkflow(
version: DatasetSearchModule.version,
inputs: [
{
key: 'datasets',
key: NodeInputKeyEnum.datasetSelectList,
renderTypeList: [FlowNodeInputTypeEnum.selectDataset, FlowNodeInputTypeEnum.reference],
label: 'core.module.input.label.Select dataset',
label: i18nT('common:core.module.input.label.Select dataset'),
value: selectedDatasets,
valueType: WorkflowIOValueTypeEnum.selectDataset,
list: [],
required: true
},
{
key: 'similarity',
key: NodeInputKeyEnum.datasetSimilarity,
renderTypeList: [FlowNodeInputTypeEnum.selectDatasetParamsModal],
label: '',
value: formData.dataset.similarity,
valueType: WorkflowIOValueTypeEnum.number
},
{
key: 'limit',
key: NodeInputKeyEnum.datasetMaxTokens,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
value: formData.dataset.limit,
valueType: WorkflowIOValueTypeEnum.number
},
{
key: 'searchMode',
key: NodeInputKeyEnum.datasetSearchMode,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.string,
value: formData.dataset.searchMode
},
{
key: 'usingReRank',
key: NodeInputKeyEnum.datasetSearchEmbeddingWeight,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.number,
value: formData.dataset.embeddingWeight
},
{
key: NodeInputKeyEnum.datasetSearchUsingReRank,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.boolean,
value: formData.dataset.usingReRank
},
{
key: 'datasetSearchUsingExtensionQuery',
key: NodeInputKeyEnum.datasetSearchRerankModel,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.string,
value: formData.dataset.rerankModel
},
{
key: NodeInputKeyEnum.datasetSearchRerankWeight,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.number,
value: formData.dataset.rerankWeight
},
{
key: NodeInputKeyEnum.datasetSearchUsingExtensionQuery,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.boolean,
value: formData.dataset.datasetSearchUsingExtensionQuery
},
{
key: 'datasetSearchExtensionModel',
key: NodeInputKeyEnum.datasetSearchExtensionModel,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.string,
value: formData.dataset.datasetSearchExtensionModel
},
{
key: 'datasetSearchExtensionBg',
key: NodeInputKeyEnum.datasetSearchExtensionBg,
renderTypeList: [FlowNodeInputTypeEnum.hidden],
label: '',
valueType: WorkflowIOValueTypeEnum.string,

View File

@@ -1,12 +1,12 @@
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
import { getRootUser } from './datas/users';
import { Call } from './utils/request';
import { getRootUser } from '@test/datas/users';
import { Call } from '@test/utils/request';
import { describe, expect, it } from 'vitest';
import handler, {
type versionListBody,
type versionListResponse
} from '../projects/app/src/pages/api/core/app/version/list';
} from '@/pages/api/core/app/version/list';
describe('app version list test', () => {
it('should return app version list', async () => {

View File

@@ -74,9 +74,9 @@ const testWorkflow = async (path: string) => {
it('Workflow test: simple workflow', async () => {
// create a simple app
await testWorkflow('projects/app/src/test/workflow/simple.json');
await testWorkflow('test/cases/workflow/simple.json');
});
it('Workflow test: output test', async () => {
console.log(await testWorkflow('projects/app/src/test/workflow/loopTest.json'));
console.log(await testWorkflow('test/cases/workflow/loopTest.json'));
});

View File

@@ -57,7 +57,7 @@ beforeAll(async () => {
type: 'replSet',
serverOptions: {
replSet: {
count: 4
count: 1
}
}
});

View File

@@ -10,7 +10,7 @@ export default defineConfig({
},
outputFile: 'test-results.json',
setupFiles: ['./test/setup.ts'],
include: ['./test/test.ts', './projects/app/**/*.test.ts'],
include: ['./test/test.ts', './test/cases/**/*.test.ts'],
testTimeout: 5000
},
resolve: {