gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v4.6.2 (#523)

Browse Source
This commit is contained in:
Archer
2023-11-28 19:28:46 +08:00
committed by GitHub
parent e765c3bf95
commit a74e1d7166
75 changed files with 1139 additions and 417 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
projects/app/src/pages/api/admin/initv46-2.ts
View File

@@ -152,6 +152,7 @@ async function init(limit: number): Promise<any> {
collectionId: data.collection_id,
q: data.q,
a: data.a,
fullTextToken: '',
indexes: [
{
defaultIndex: !data.a,

80
projects/app/src/pages/api/admin/initv462-2.ts Normal file
View File

@@ -0,0 +1,80 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@/utils/tools';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
import { ModuleDataTypeEnum, ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants';
import { ModuleItemType } from '@fastgpt/global/core/module/type';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
console.log('total', await MongoApp.countDocuments());
await initApp(limit);
jsonRes(res, {
message: 'success'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function initApp(limit = 50): Promise<any> {
try {
const apps = await MongoApp.find({ inited: false }).limit(limit);
if (apps.length === 0) return;
const result = await Promise.allSettled(
apps.map(async (app) => {
// 遍历app的modules，找到 datasetSearch, 如果 rerank=true， searchMode = embFullTextReRank, 否则等于embedding
const modules = JSON.parse(JSON.stringify(app.modules)) as ModuleItemType[];
modules.forEach((module) => {
if (module.flowType === FlowNodeTypeEnum.datasetSearchNode) {
module.inputs.forEach((input, i) => {
if (input.key === 'rerank') {
const val = !!input.value as boolean;
module.inputs.splice(i, 1, {
key: ModuleInputKeyEnum.datasetSearchMode,
type: FlowNodeInputTypeEnum.hidden,
label: 'core.dataset.search.Mode',
valueType: ModuleDataTypeEnum.string,
showTargetInApp: false,
showTargetInPlugin: false,
value: val
? DatasetSearchModeEnum.embFullTextReRank
: DatasetSearchModeEnum.embedding
});
}
});
}
});
app.modules = modules;
app.inited = true;
await app.save();
})
);
success += result.filter((item) => item.status === 'fulfilled').length;
console.log(`success: ${success}`);
return initApp(limit);
} catch (error) {
console.log(error);
await delay(1000);
return initApp(limit);
}
}

27
projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
View File

@@ -8,6 +8,7 @@ import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d';
import type { ModuleItemType } from '@fastgpt/global/core/module/type';
import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { FormatForm2ModulesProps } from '@fastgpt/global/core/app/api';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
@@ -378,7 +379,7 @@ function datasetTemplate({
},
{
key: 'similarity',
value: 0.5,
value: 0.4,
type: FlowNodeInputTypeEnum.slider,
label: '相似度',
connected: true
@@ -403,13 +404,23 @@ function datasetTemplate({
connected: true
},
{
key: 'rerank',
type: FlowNodeInputTypeEnum.switch,
label: '结果重排',
description: '将召回的结果进行进一步重排，可增加召回率',
plusField: true,
connected: true,
value: true
key: 'searchMode',
type: 'hidden',
label: 'core.dataset.search.Mode',
valueType: 'string',
showTargetInApp: false,
showTargetInPlugin: false,
value: DatasetSearchModeEnum.embFullTextReRank,
connected: false
},
{
key: 'datasetParamsModal',
type: 'selectDatasetParamsModal',
label: '',
connected: false,
valueType: 'any',
showTargetInApp: false,
showTargetInPlugin: false
}
],
outputs: [

22
projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts
View File

@@ -312,13 +312,23 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
connected: true
},
{
key: 'rerank',
type: FlowNodeInputTypeEnum.switch,
label: '结果重排',
description: '将召回的结果进行进一步重排，可增加召回率',
plusField: true,
key: 'searchMode',
type: 'hidden',
label: 'core.dataset.search.Mode',
valueType: 'string',
showTargetInApp: false,
showTargetInPlugin: false,
value: formData.dataset.searchMode,
connected: false
},
{
key: 'datasetParamsModal',
type: 'selectDatasetParamsModal',
label: '',
connected: false,
value: formData.dataset.rerank
valueType: 'any',
showTargetInApp: false,
showTargetInPlugin: false
}
],
outputs: [

1
projects/app/src/pages/api/core/app/update.ts
View File

@@ -4,7 +4,6 @@ import { connectToDatabase } from '@/service/mongo';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import type { AppUpdateParams } from '@fastgpt/global/core/app/api';
import { authApp } from '@fastgpt/service/support/permission/auth/app';
import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants';
/* 获取我的模型 */
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

7
projects/app/src/pages/api/core/dataset/exportAll.ts
View File

@@ -67,8 +67,15 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
'limit.exportKbTime': new Date()
});
});
cursor.on('error', (err) => {
addLog.error(`export dataset error`, err);
res.status(500);
res.end();
});
} catch (err) {
res.status(500);
addLog.error(`export dataset error`, err);
jsonRes(res, {
code: 500,
error: err

4
projects/app/src/pages/api/core/dataset/searchTest.ts
View File

@@ -15,7 +15,7 @@ import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { datasetId, text, limit = 20, rerank } = req.body as SearchTestProps;
const { datasetId, text, limit = 20, searchMode } = req.body as SearchTestProps;
if (!datasetId || !text) {
throw new Error('缺少参数');
@@ -40,7 +40,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
model: dataset.vectorModel,
limit: Math.min(limit, 50),
datasetIds: [datasetId],
rerank
searchMode
});
// push bill

26
projects/app/src/pages/api/system/getInitData.ts
View File

@@ -14,7 +14,8 @@ import {
defaultQGModels,
defaultVectorModels,
defaultAudioSpeechModels,
defaultWhisperModel
defaultWhisperModel,
defaultReRankModels
} from '@fastgpt/global/core/ai/model';
import { SimpleModeTemplate_FastGPT_Universal } from '@/global/core/app/constants';
import { getSimpleTemplatesFromPlus } from '@/service/core/app/utils';
@@ -31,11 +32,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
cqModels: global.cqModels,
extractModels: global.extractModels,
vectorModels: global.vectorModels,
audioSpeechModels: global.audioSpeechModels.map((item) => ({
reRankModels: global.reRankModels.map((item) => ({
...item,
baseUrl: undefined,
key: undefined
requestUrl: undefined,
requestAuth: undefined
})),
audioSpeechModels: global.audioSpeechModels,
priceMd: global.priceMd,
systemVersion: global.systemVersion || '0.0.0',
simpleModeTemplates: global.simpleModeTemplates
@@ -50,12 +52,11 @@ const defaultSystemEnv: SystemEnvType = {
};
const defaultFeConfigs: FeConfigsType = {
show_emptyChat: true,
show_contact: true,
show_git: true,
docUrl: 'https://docs.fastgpt.in',
show_register: false,
docUrl: 'https://doc.fastgpt.in',
openAPIDocUrl: 'https://doc.fastgpt.in/docs/development/openapi',
systemTitle: 'FastGPT',
authorText: 'Made by FastGPT Team.',
limit: {
exportLimitMinutes: 0
},
@@ -99,7 +100,14 @@ export function setDefaultData(res?: ConfigFileType) {
? { ...defaultSystemEnv, ...res.SystemParams }
: defaultSystemEnv;
global.feConfigs = res?.FeConfig
? { ...defaultFeConfigs, ...res.FeConfig, isPlus: !!res.SystemParams?.pluginBaseUrl }
? {
concatMd: res?.FeConfig?.show_git
? '* 项目开源地址: [FastGPT GitHub](https://github.com/labring/FastGPT)\n* 交流群: ![](https://doc.fastgpt.in/wechat-fastgpt.webp)'
: '',
...defaultFeConfigs,
...res.FeConfig,
isPlus: !!res.SystemParams?.pluginBaseUrl
}
: defaultFeConfigs;
global.chatModels = res?.ChatModels || defaultChatModels;
@@ -110,6 +118,8 @@ export function setDefaultData(res?: ConfigFileType) {
global.vectorModels = res?.VectorModels || defaultVectorModels;
global.reRankModels = res?.ReRankModels || defaultReRankModels;
global.audioSpeechModels = res?.AudioSpeechModels || defaultAudioSpeechModels;
global.whisperModel = res?.WhisperModel || defaultWhisperModel;

2
projects/app/src/pages/api/v1/embeddings.ts
View File

@@ -15,7 +15,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
try {
let { input, model, billId } = req.body as Props;
await connectToDatabase();
const { teamId, tmbId } = await authCert({ req, authToken: true });
const { teamId, tmbId } = await authCert({ req, authToken: true, authApiKey: true });
if (!Array.isArray(input) || typeof input !== 'string') {
throw new Error('input is nor array or string');

42
projects/app/src/pages/api/v1/rerank.ts Normal file
View File

@@ -0,0 +1,42 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { pushReRankBill } from '@/service/support/wallet/bill/push';
import { connectToDatabase } from '@/service/mongo';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { PostReRankProps } from '@fastgpt/global/core/ai/api';
import { reRankRecall } from '@/service/core/ai/rerank';
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
let { query, inputs } = req.body as PostReRankProps;
await connectToDatabase();
const { teamId, tmbId } = await authCert({
req,
authApiKey: true
});
await authTeamBalance(teamId);
// max 150 length
inputs = inputs.slice(0, 150);
const result = await reRankRecall({ query, inputs });
pushReRankBill({
teamId,
tmbId,
source: 'api'
});
jsonRes(res, {
data: result
});
} catch (err) {
console.log(err);
jsonRes(res, {
code: 500,
error: err
});
}
});