V4.6.6-2 (#673)

2023-12-31 14:12:51 +08:00
parent ccca0468da
commit 9ccfda47b7
270 changed files with 8182 additions and 1295 deletions
--- a/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts
+++ b/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts
@@ -19,7 +19,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<

    const qgModel = global.qgModels[0];

-    const { result, tokens } = await createQuestionGuide({
+    const { result, inputTokens, outputTokens } = await createQuestionGuide({
      messages,
      model: qgModel.model
    });
@@ -29,7 +29,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    });

    pushQuestionGuideBill({
-      tokens: tokens,
+      inputTokens,
+      outputTokens,
      teamId,
      tmbId
    });
--- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
+++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
@@ -374,11 +374,21 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
        {
          key: 'searchMode',
          type: 'hidden',
-          label: 'core.dataset.search.Mode',
+          label: '',
          valueType: 'string',
          showTargetInApp: false,
          showTargetInPlugin: false,
-          value: DatasetSearchModeEnum.embFullTextReRank,
+          value: DatasetSearchModeEnum.mixedRecall,
+          connected: false
+        },
+        {
+          key: 'usingReRank',
+          type: 'hidden',
+          label: '',
+          valueType: 'string',
+          showTargetInApp: false,
+          showTargetInPlugin: false,
+          value: true,
          connected: false
        },
        {
--- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts
+++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts
@@ -377,6 +377,16 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
          value: formData.dataset.searchMode,
          connected: false
        },
+        {
+          key: 'usingReRank',
+          type: 'hidden',
+          label: '',
+          valueType: 'string',
+          showTargetInApp: false,
+          showTargetInPlugin: false,
+          value: formData.dataset.usingReRank,
+          connected: false
+        },
        {
          key: 'datasetParamsModal',
          type: 'selectDatasetParamsModal',
--- a/projects/app/src/pages/api/core/chat/item/getSpeech.ts
+++ b/projects/app/src/pages/api/core/chat/item/getSpeech.ts
@@ -56,7 +56,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
        try {
          pushAudioSpeechBill({
            model: model,
-            textLength: input.length,
+            textLen: input.length,
            tmbId,
            teamId,
            source: authType2BillSource({ authType })
--- a/projects/app/src/pages/api/core/dataset/collection/list.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/list.ts
@@ -6,10 +6,7 @@ import { Types } from '@fastgpt/service/common/mongo';
 import type { DatasetCollectionsListItemType } from '@/global/core/dataset/type.d';
 import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
 import { PagingData } from '@/types';
-import {
-  DatasetColCollectionName,
-  MongoDatasetCollection
-} from '@fastgpt/service/core/dataset/collection/schema';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import { startQueue } from '@/service/utils/tools';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
--- a/projects/app/src/pages/api/core/dataset/data/insertData.ts
+++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts
@@ -69,7 +69,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      a: formatA
    });

-    const { insertId, tokenLen } = await insertData2Dataset({
+    const { insertId, tokens } = await insertData2Dataset({
      teamId,
      tmbId,
      datasetId,
@@ -84,7 +84,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
    pushGenerateVectorBill({
      teamId,
      tmbId,
-      tokenLen: tokenLen,
+      tokens,
      model: vectorModelData.model
    });

--- a/projects/app/src/pages/api/core/dataset/data/update.ts
+++ b/projects/app/src/pages/api/core/dataset/data/update.ts
@@ -30,7 +30,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
    // auth team balance
    await authTeamBalance(teamId);

-    const { tokenLen } = await updateData2Dataset({
+    const { tokens } = await updateData2Dataset({
      dataId: id,
      q,
      a,
@@ -38,14 +38,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      model: vectorModel
    });

-    if (tokenLen) {
-      pushGenerateVectorBill({
-        teamId,
-        tmbId,
-        tokenLen: tokenLen,
-        model: vectorModel
-      });
-    }
+    pushGenerateVectorBill({
+      teamId,
+      tmbId,
+      tokens,
+      model: vectorModel
+    });

    jsonRes(res);
  } catch (err) {
--- a/projects/app/src/pages/api/core/dataset/searchTest.ts
+++ b/projects/app/src/pages/api/core/dataset/searchTest.ts
@@ -6,7 +6,7 @@ import { connectToDatabase } from '@/service/mongo';
 import { authDataset } from '@fastgpt/service/support/permission/auth/dataset';
 import { authTeamBalance } from '@/service/support/permission/auth/bill';
 import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
-import { searchDatasetData } from '@/service/core/dataset/data/pg';
+import { searchDatasetData } from '@/service/core/dataset/data/controller';
 import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
 import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
 import { searchQueryExtension } from '@fastgpt/service/core/ai/functions/queryExtension';
@@ -14,7 +14,7 @@ import { searchQueryExtension } from '@fastgpt/service/core/ai/functions/queryEx
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
    await connectToDatabase();
-    const { datasetId, text, limit = 20, searchMode } = req.body as SearchTestProps;
+    const { datasetId, text, limit = 20, searchMode, usingReRank } = req.body as SearchTestProps;

    if (!datasetId || !text) {
      throw new Error('缺少参数');
@@ -40,20 +40,21 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
    //   model: global.chatModels[0].model
    // });

-    const { searchRes, tokenLen } = await searchDatasetData({
+    const { searchRes, tokens } = await searchDatasetData({
      rawQuery: text,
      queries: [text],
      model: dataset.vectorModel,
      limit: Math.min(limit * 800, 30000),
      datasetIds: [datasetId],
-      searchMode
+      searchMode,
+      usingReRank
    });

    // push bill
    const { total } = pushGenerateVectorBill({
      teamId,
      tmbId,
-      tokenLen: tokenLen,
+      tokens,
      model: dataset.vectorModel,
      source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt
    });
--- a/projects/app/src/pages/api/core/dataset/training/getQueueLen.ts
+++ b/projects/app/src/pages/api/core/dataset/training/getQueueLen.ts
@@ -3,20 +3,39 @@ import { jsonRes } from '@fastgpt/service/common/response';
 import { connectToDatabase } from '@/service/mongo';
 import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { GetTrainingQueueProps } from '@/global/core/dataset/api';

-/* 拆分数据成QA */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
  try {
    await connectToDatabase();
    await authCert({ req, authToken: true });
+    const { vectorModel, agentModel } = req.query as GetTrainingQueueProps;

-    // split queue data
-    const result = await MongoDatasetTraining.countDocuments({
-      lockTime: { $lt: new Date('2040/1/1') }
-    });
+    // get queue data
+    // 分别统计 model = vectorModel和agentModel的数量
+    const data = await MongoDatasetTraining.aggregate([
+      {
+        $match: {
+          lockTime: { $lt: new Date('2040/1/1') },
+          $or: [{ model: { $eq: vectorModel } }, { model: { $eq: agentModel } }]
+        }
+      },
+      {
+        $group: {
+          _id: '$model',
+          count: { $sum: 1 }
+        }
+      }
+    ]);
+
+    const vectorTrainingCount = data.find((item) => item._id === vectorModel)?.count || 0;
+    const agentTrainingCount = data.find((item) => item._id === agentModel)?.count || 0;

    jsonRes(res, {
-      data: result
+      data: {
+        vectorTrainingCount,
+        agentTrainingCount
+      }
    });
  } catch (err) {
    jsonRes(res, {