gitea/FastGPT
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

V4.6.6-2 (#673)

Browse Source
This commit is contained in:
Archer
2023-12-31 14:12:51 +08:00
committed by GitHub
parent ccca0468da
commit 9ccfda47b7
270 changed files with 8182 additions and 1295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
projects/app/src/pages/api/admin/initv46-2.ts
View File

@@ -2,11 +2,9 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/pg';
import {
DatasetDataIndexTypeEnum,
PgDatasetTableName
} from '@fastgpt/global/core/dataset/constant';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';

4
projects/app/src/pages/api/admin/initv46-fix.ts
View File

@@ -2,8 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { PgClient } from '@fastgpt/service/common/pg';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';

4
projects/app/src/pages/api/admin/initv46.ts
View File

@@ -13,8 +13,8 @@ import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { PgClient } from '@fastgpt/service/common/pg';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
import { MongoOpenApi } from '@fastgpt/service/support/openapi/schema';
import { MongoApp } from '@fastgpt/service/core/app/schema';

80
projects/app/src/pages/api/admin/initv462-2.ts
View File

@@ -1,80 +0,0 @@
import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoApp } from '@fastgpt/service/core/app/schema';
import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant';
import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant';
import { ModuleIOValueTypeEnum, ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants';
import { ModuleItemType } from '@fastgpt/global/core/module/type';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const { limit = 50 } = req.body as { limit: number };
await authCert({ req, authRoot: true });
await connectToDatabase();
success = 0;
console.log('total', await MongoApp.countDocuments());
await initApp(limit);
jsonRes(res, {
message: 'success'
});
} catch (error) {
console.log(error);
jsonRes(res, {
code: 500,
error
});
}
}
export async function initApp(limit = 50): Promise<any> {
try {
const apps = await MongoApp.find({ inited: false }).limit(limit);
if (apps.length === 0) return;
const result = await Promise.allSettled(
apps.map(async (app) => {
// 遍历app的modules，找到 datasetSearch, 如果 rerank=true， searchMode = embFullTextReRank, 否则等于embedding
const modules = JSON.parse(JSON.stringify(app.modules)) as ModuleItemType[];
modules.forEach((module) => {
if (module.flowType === FlowNodeTypeEnum.datasetSearchNode) {
module.inputs.forEach((input, i) => {
if (input.key === 'rerank') {
const val = !!input.value as boolean;
module.inputs.splice(i, 1, {
key: ModuleInputKeyEnum.datasetSearchMode,
type: FlowNodeInputTypeEnum.hidden,
label: 'core.dataset.search.Mode',
valueType: ModuleIOValueTypeEnum.string,
showTargetInApp: false,
showTargetInPlugin: false,
value: val
? DatasetSearchModeEnum.embFullTextReRank
: DatasetSearchModeEnum.embedding
});
}
});
}
});
app.modules = modules;
app.inited = true;
await app.save();
})
);
success += result.filter((item) => item.status === 'fulfilled').length;
console.log(`success: ${success}`);
return initApp(limit);
} catch (error) {
console.log(error);
await delay(1000);
return initApp(limit);
}
}

2
projects/app/src/pages/api/admin/initv462.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { jiebaSplit } from '@/service/core/dataset/utils';
import { jiebaSplit } from '@/service/common/string/jieba';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */

2
projects/app/src/pages/api/admin/initv463-2.ts
View File

@@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo';
import { delay } from '@fastgpt/global/common/system/utils';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
import { jiebaSplit } from '@/service/core/dataset/utils';
import { jiebaSplit } from '@/service/common/string/jieba';
let success = 0;
/* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */

4
projects/app/src/pages/api/admin/initv464.ts
View File

@@ -2,8 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { PgClient } from '@fastgpt/service/common/pg';
import { PgDatasetTableName } from '@fastgpt/global/core/dataset/constant';
import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
import { PgDatasetTableName } from '@fastgpt/global/common/vectorStore/constants';
import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
import { connectToDatabase } from '@/service/mongo';

58
projects/app/src/pages/api/common/system/getInitData.ts
View File

@@ -4,7 +4,6 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { readFileSync, readdirSync } from 'fs';
import type { InitDateResponse } from '@/global/common/api/systemRes';
import type { FastGPTConfigFileType } from '@fastgpt/global/common/system/types/index.d';
import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools';
import { getTikTokenEnc } from '@fastgpt/global/common/string/tiktoken';
import { initHttpAgent } from '@fastgpt/service/common/middle/httpAgent';
import { SimpleModeTemplate_FastGPT_Universal } from '@/global/core/app/constants';
@@ -33,8 +32,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
requestUrl: undefined,
requestAuth: undefined
})) || [],
qgModes: global.qgModels,
whisperModel: global.whisperModel,
audioSpeechModels: global.audioSpeechModels,
priceMd: global.priceMd,
systemVersion: global.systemVersion || '0.0.0',
simpleModeTemplates: global.simpleModeTemplates
}
@@ -73,7 +73,6 @@ export async function getInitConfig() {
await getSimpleModeTemplates();
getSystemVersion();
countModelPrice();
getSystemPlugin();
console.log({
@@ -88,7 +87,6 @@ export async function getInitConfig() {
reRankModels: global.reRankModels,
audioSpeechModels: global.audioSpeechModels,
whisperModel: global.whisperModel,
price: global.priceMd,
simpleModeTemplates: global.simpleModeTemplates,
communityPlugins: global.communityPlugins
});
@@ -123,22 +121,20 @@ export async function initSystemConfig() {
// set config
global.feConfigs = {
isPlus: !!config.systemEnv.pluginBaseUrl,
isPlus: !!config.systemEnv?.pluginBaseUrl,
...config.feConfigs
};
global.systemEnv = config.systemEnv;
global.chatModels = config.chatModels || [];
global.qaModels = config.qaModels || [];
global.cqModels = config.cqModels || [];
global.extractModels = config.extractModels || [];
global.qgModels = config.qgModels || [];
global.vectorModels = config.vectorModels || [];
global.reRankModels = config.reRankModels || [];
global.audioSpeechModels = config.audioSpeechModels || [];
global.chatModels = config.chatModels;
global.qaModels = config.qaModels;
global.cqModels = config.cqModels;
global.extractModels = config.extractModels;
global.qgModels = config.qgModels;
global.vectorModels = config.vectorModels;
global.reRankModels = config.reRankModels;
global.audioSpeechModels = config.audioSpeechModels;
global.whisperModel = config.whisperModel;
global.priceMd = '';
}
export function initGlobal() {
@@ -168,38 +164,6 @@ export function getSystemVersion() {
}
}
export function countModelPrice() {
global.priceMd = `| 计费项 | 价格: 元/ 1K tokens(包含上下文)|
| --- | --- |
${global.vectorModels
?.map((item) => `| 索引-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${global.chatModels
?.map((item) => `| 对话-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${global.qaModels
?.map((item) => `| 文件QA拆分-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${global.cqModels
?.map((item) => `| 问题分类-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${global.extractModels
?.map((item) => `| 内容提取-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${global.qgModels
?.map((item) => `| 下一步指引-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${global.audioSpeechModels
?.map((item) => `| 语音播放-${item.name} | ${formatPrice(item.price, 1000)} |`)
.join('\n')}
${
global.whisperModel
? `| 语音输入-${global.whisperModel.name} | ${global.whisperModel.price}/分钟 |`
: ''
}
`;
}
async function getSimpleModeTemplates() {
if (global.simpleModeTemplates && global.simpleModeTemplates.length > 0) return;

6
projects/app/src/pages/api/common/system/refreshConfig.ts
View File

@@ -2,14 +2,13 @@ import type { NextApiRequest, NextApiResponse } from 'next';
import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { countModelPrice, initSystemConfig } from './getInitData';
import { initSystemConfig } from './getInitData';
export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
await authCert({ req, authRoot: true });
await initSystemConfig();
countModelPrice();
console.log(`refresh config`);
console.log({
@@ -23,8 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
vectorModels: global.vectorModels,
reRankModels: global.reRankModels,
audioSpeechModels: global.audioSpeechModels,
whisperModel: global.whisperModel,
price: global.priceMd
whisperModel: global.whisperModel
});
} catch (error) {
console.log(error);

5
projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts
View File

@@ -19,7 +19,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
const qgModel = global.qgModels[0];
const { result, tokens } = await createQuestionGuide({
const { result, inputTokens, outputTokens } = await createQuestionGuide({
messages,
model: qgModel.model
});
@@ -29,7 +29,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
});
pushQuestionGuideBill({
tokens: tokens,
inputTokens,
outputTokens,
teamId,
tmbId
});

14
projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
View File

@@ -374,11 +374,21 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
{
key: 'searchMode',
type: 'hidden',
label: 'core.dataset.search.Mode',
label: '',
valueType: 'string',
showTargetInApp: false,
showTargetInPlugin: false,
value: DatasetSearchModeEnum.embFullTextReRank,
value: DatasetSearchModeEnum.mixedRecall,
connected: false
},
{
key: 'usingReRank',
type: 'hidden',
label: '',
valueType: 'string',
showTargetInApp: false,
showTargetInPlugin: false,
value: true,
connected: false
},
{

10
projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts
View File

@@ -377,6 +377,16 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
value: formData.dataset.searchMode,
connected: false
},
{
key: 'usingReRank',
type: 'hidden',
label: '',
valueType: 'string',
showTargetInApp: false,
showTargetInPlugin: false,
value: formData.dataset.usingReRank,
connected: false
},
{
key: 'datasetParamsModal',
type: 'selectDatasetParamsModal',

2
projects/app/src/pages/api/core/chat/item/getSpeech.ts
View File

@@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
try {
pushAudioSpeechBill({
model: model,
textLength: input.length,
textLen: input.length,
tmbId,
teamId,
source: authType2BillSource({ authType })

5
projects/app/src/pages/api/core/dataset/collection/list.ts
View File

@@ -6,10 +6,7 @@ import { Types } from '@fastgpt/service/common/mongo';
import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
import { PagingData } from '@/types';
import {
DatasetColCollectionName,
MongoDatasetCollection
} from '@fastgpt/service/core/dataset/collection/schema';
import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
import { startQueue } from '@/service/utils/tools';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';

4
projects/app/src/pages/api/core/dataset/data/insertData.ts
View File

@@ -69,7 +69,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
a: formatA
});
const { insertId, tokenLen } = await insertData2Dataset({
const { insertId, tokens } = await insertData2Dataset({
teamId,
tmbId,
datasetId,
@@ -84,7 +84,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
tokens,
model: vectorModelData.model
});

16
projects/app/src/pages/api/core/dataset/data/update.ts
View File

@@ -30,7 +30,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// auth team balance
await authTeamBalance(teamId);
const { tokenLen } = await updateData2Dataset({
const { tokens } = await updateData2Dataset({
dataId: id,
q,
a,
@@ -38,14 +38,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
model: vectorModel
});
if (tokenLen) {
pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
model: vectorModel
});
}
pushGenerateVectorBill({
teamId,
tmbId,
tokens,
model: vectorModel
});
jsonRes(res);
} catch (err) {

11
projects/app/src/pages/api/core/dataset/searchTest.ts
View File

@@ -6,7 +6,7 @@ import { connectToDatabase } from '@/service/mongo';
import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
import { searchDatasetData } from '@/service/core/dataset/data/pg';
import { searchDatasetData } from '@/service/core/dataset/data/controller';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
import { searchQueryExtension } from '@fastgpt/service/core/ai/functions/queryExtension';
@@ -14,7 +14,7 @@ import { searchQueryExtension } from '@fastgpt/service/core/ai/functions/queryEx
export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
try {
await connectToDatabase();
const { datasetId, text, limit = 20, searchMode } = req.body as SearchTestProps;
const { datasetId, text, limit = 20, searchMode, usingReRank } = req.body as SearchTestProps;
if (!datasetId || !text) {
throw new Error('缺少参数');
@@ -40,20 +40,21 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
// model: global.chatModels[0].model
// });
const { searchRes, tokenLen } = await searchDatasetData({
const { searchRes, tokens } = await searchDatasetData({
rawQuery: text,
queries: [text],
model: dataset.vectorModel,
limit: Math.min(limit * 800, 30000),
datasetIds: [datasetId],
searchMode
searchMode,
usingReRank
});
// push bill
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokenLen: tokenLen,
tokens,
model: dataset.vectorModel,
source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt
});

31
projects/app/src/pages/api/core/dataset/training/getQueueLen.ts
View File

@@ -3,20 +3,39 @@ import { jsonRes } from '@fastgpt/service/common/response';
import { connectToDatabase } from '@/service/mongo';
import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
import { authCert } from '@fastgpt/service/support/permission/auth/common';
import { GetTrainingQueueProps } from '@/global/core/dataset/api';
/* 拆分数据成QA */
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
await connectToDatabase();
await authCert({ req, authToken: true });
const { vectorModel, agentModel } = req.query as GetTrainingQueueProps;
// split queue data
const result = await MongoDatasetTraining.countDocuments({
lockTime: { $lt: new Date('2040/1/1') }
});
// get queue data
// 分别统计 model = vectorModel和agentModel的数量
const data = await MongoDatasetTraining.aggregate([
{
$match: {
lockTime: { $lt: new Date('2040/1/1') },
$or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }]
}
},
{
$group: {
_id: '$model',
count: { $sum: 1 }
}
}
]);
const vectorTrainingCount = data.find((item) => item._id === vectorModel)?.count || 0;
const agentTrainingCount = data.find((item) => item._id === agentModel)?.count || 0;
jsonRes(res, {
data: result
data: {
vectorTrainingCount,
agentTrainingCount
}
});
} catch (err) {
jsonRes(res, {

10
projects/app/src/pages/api/v1/embeddings.ts
View File

@@ -5,7 +5,7 @@ import { withNextCors } from '@fastgpt/service/common/middle/cors';
import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
import { connectToDatabase } from '@/service/mongo';
import { authTeamBalance } from '@/service/support/permission/auth/bill';
import { getVectorsByText, GetVectorProps } from '@/service/core/ai/vector';
import { getVectorsByText, GetVectorProps } from '@fastgpt/service/core/ai/embedding';
import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
import { getBillSourceByAuthType } from '@fastgpt/global/support/wallet/bill/tools';
@@ -30,7 +30,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
await authTeamBalance(teamId);
const { tokenLen, vectors } = await getVectorsByText({ input, model });
const { tokens, vectors } = await getVectorsByText({ input, model });
jsonRes(res, {
data: {
@@ -42,8 +42,8 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
})),
model,
usage: {
prompt_tokens: tokenLen,
total_tokens: tokenLen
prompt_tokens: tokens,
total_tokens: tokens
}
}
});
@@ -51,7 +51,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
const { total } = pushGenerateVectorBill({
teamId,
tmbId,
tokenLen,
tokens,
model,
billId,
source: getBillSourceByAuthType({ authType })

8
projects/app/src/pages/api/v1/rerank.ts
View File

@@ -42,11 +42,9 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
data: result
});
} catch (err) {
console.log(err);
jsonRes<PostReRankResponse>(res, {
data: inputs.map((input) => ({
id: input.id
}))
jsonRes(res, {
code: 500,
error: err
});
}
});