gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: import dataset step error;perf: ai proxy avatar (#4074)

Browse Source 
* perf: pg config params

* perf: ai proxy avatar

* fix: import dataset step error

* feat: data input ux

* perf: app dataset rewite
This commit is contained in:
Archer
2025-03-11 14:04:48 +08:00
committed by archer
parent 92b2ecc381
commit 3f794baf2e
37 changed files with 503 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
packages/service/common/vectorStore/pg/class.ts
View File

@@ -38,6 +38,27 @@ export class PgVectorCtrl {
await PgClient.query(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS create_time_index ON ${DatasetVectorTableName} USING btree(createtime);`
);
// 10w rows
// await PgClient.query(`
// ALTER TABLE modeldata SET (
// autovacuum_vacuum_scale_factor = 0.1,
// autovacuum_analyze_scale_factor = 0.05,
// autovacuum_vacuum_threshold = 50,
// autovacuum_analyze_threshold = 50,
// autovacuum_vacuum_cost_delay = 20,
// autovacuum_vacuum_cost_limit = 200
// );`);
// 100w rows
// await PgClient.query(`
// ALTER TABLE modeldata SET (
// autovacuum_vacuum_scale_factor = 0.01,
// autovacuum_analyze_scale_factor = 0.02,
// autovacuum_vacuum_threshold = 1000,
// autovacuum_analyze_threshold = 1000,
// autovacuum_vacuum_cost_delay = 10,
// autovacuum_vacuum_cost_limit = 2000
// );`)
addLog.info('init pg successful');
} catch (error) {

132
packages/service/core/app/utils.ts
View File

@@ -4,20 +4,13 @@ import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
import type { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
export type ListByAppIdAndDatasetIdsBody = {
teamId: string;
datasetIdList: string[];
};
interface Dataset {
datasetId: string;
[key: string]: any;
}
export async function listAppDatasetDataByTeamIdAndDatasetIds({
teamId,
datasetIdList
}: ListByAppIdAndDatasetIdsBody) {
}: {
teamId?: string;
datasetIdList: string[];
}) {
const myDatasets = await MongoDataset.find({
teamId,
_id: { $in: datasetIdList }
@@ -31,9 +24,18 @@ export async function listAppDatasetDataByTeamIdAndDatasetIds({
}));
}
export async function rewriteAppWorkflowToDetail(nodes: StoreNodeItemType[], teamId: string) {
export async function rewriteAppWorkflowToDetail({
nodes,
teamId,
isRoot
}: {
nodes: StoreNodeItemType[];
teamId: string;
isRoot: boolean;
}) {
const datasetIdSet = new Set<string>();
// Get all dataset ids from nodes
nodes.forEach((node) => {
if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return;
@@ -41,83 +43,83 @@ export async function rewriteAppWorkflowToDetail(nodes: StoreNodeItemType[], tea
if (!input) return;
const rawValue = input.value as undefined | { datasetId: string }[] | { datasetId: string };
if (!rawValue) return;
const datasetIds = Array.isArray(rawValue)
? rawValue
.map((v) => v?.datasetId)
.filter((id): id is string => !!id && typeof id === 'string')
: rawValue?.datasetId
? rawValue.map((v) => v?.datasetId).filter((id) => !!id && typeof id === 'string')
: rawValue.datasetId
? [String(rawValue.datasetId)]
: [];
if (datasetIds.length === 0) return;
datasetIds.forEach((id) => datasetIdSet.add(id));
});
if (datasetIdSet.size === 0) return;
const uniqueDatasetIds = Array.from(datasetIdSet);
// Load dataset list
const datasetList = await listAppDatasetDataByTeamIdAndDatasetIds({
teamId,
datasetIdList: uniqueDatasetIds
teamId: isRoot ? undefined : teamId,
datasetIdList: Array.from(datasetIdSet)
});
const datasetMap = new Map(datasetList.map((ds) => [String(ds.datasetId), ds]));
const datasetMap = new Map(
datasetList.map((ds) => [
String(ds.datasetId),
{
...ds,
vectorModel: getEmbeddingModel(ds.vectorModel.model)
}
])
);
// Rewrite dataset ids, add dataset info to nodes
nodes.forEach((node) => {
if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return;
const input = node.inputs.find((item) => item.key === NodeInputKeyEnum.datasetSelectList);
if (!input) return;
node.inputs.forEach((item) => {
if (item.key !== NodeInputKeyEnum.datasetSelectList) return;
const rawValue = input.value as undefined | { datasetId: string }[] | { datasetId: string };
const val = item.value as undefined | { datasetId: string }[] | { datasetId: string };
const datasetIds = Array.isArray(rawValue)
? rawValue
.map((v) => v?.datasetId)
.filter((id): id is string => !!id && typeof id === 'string')
: rawValue?.datasetId
? [String(rawValue.datasetId)]
: [];
if (datasetIds.length === 0) return;
input.value = datasetIds
.map((id) => {
const data = datasetMap.get(String(id));
return data
? {
datasetId: data.datasetId,
avatar: data.avatar,
name: data.name,
vectorModel: data.vectorModel
}
: undefined;
})
.filter((item): item is NonNullable<typeof item> => !!item);
if (Array.isArray(val)) {
item.value = val.map((v) => {
const data = datasetMap.get(String(v.datasetId))!;
return {
datasetId: data.datasetId,
avatar: data.avatar,
name: data.name,
vectorModel: data.vectorModel
};
});
} else if (typeof val === 'object' && val !== null) {
const data = datasetMap.get(String(val.datasetId))!;
item.value = {
datasetId: data.datasetId,
avatar: data.avatar,
name: data.name,
vectorModel: data.vectorModel
};
}
});
});
return nodes;
}
export async function rewriteAppWorkflowToSimple(formatNodes: StoreNodeItemType[]) {
formatNodes.forEach((node) => {
if (node.flowNodeType !== FlowNodeTypeEnum.datasetSearchNode) return;
const datasetsInput = node.inputs.find(
(input) => input.key === NodeInputKeyEnum.datasetSelectList
);
if (datasetsInput?.value) {
datasetsInput.value = datasetsInput.value.map((dataset: Dataset) => ({
datasetId: dataset.datasetId
}));
}
node.inputs.forEach((input) => {
if (input.key === NodeInputKeyEnum.datasetSelectList) {
const val = input.value as undefined | { datasetId: string }[] | { datasetId: string };
if (!val) {
input.value = [];
} else if (Array.isArray(val)) {
input.value = val
.map((dataset: { datasetId: string }) => ({
datasetId: dataset.datasetId
}))
.filter((item) => !!item.datasetId);
} else if (typeof val === 'object' && val !== null) {
input.value = [
{
datasetId: val.datasetId
}
];
}
}
});
});
}