4.6.8-production (#822)

* Json completion (#16) * json-completion * fix duplicate * fix * fix: config json * feat: query extension * perf: i18n * 468 doc * json editor * perf: doc * perf: default extension model * docker file * doc * perf: token count * perf: search extension * format * perf: some constants data --------- Co-authored-by: heheer <71265218+newfish-cmyk@users.noreply.github.com>
2024-02-05 00:51:46 +08:00
parent ec8e2512bc
commit 51bbdf26a3
68 changed files with 4118 additions and 3787 deletions
--- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
+++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-simple.ts
@@ -294,7 +294,7 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
          valueType: 'string',
          targets: [
            {
-              moduleId: 'vuc92c',
+              moduleId: 'datasetSearch',
              key: 'userChatInput'
            }
          ]
@@ -387,15 +387,6 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
          value: true,
          connected: false
        },
-        {
-          key: 'datasetParamsModal',
-          type: 'selectDatasetParamsModal',
-          label: '',
-          valueType: 'any',
-          showTargetInApp: false,
-          showTargetInPlugin: false,
-          connected: false
-        },
        {
          key: 'userChatInput',
          type: 'target',
@@ -495,19 +486,6 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
          label: '温度',
          value: 0,
          valueType: 'number',
-          min: 0,
-          max: 10,
-          step: 1,
-          markList: [
-            {
-              label: '严谨',
-              value: 0
-            },
-            {
-              label: '发散',
-              value: 10
-            }
-          ],
          showTargetInApp: false,
          showTargetInPlugin: false,
          connected: false
@@ -518,19 +496,6 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
          label: '回复上限',
          value: maxToken,
          valueType: 'number',
-          min: 100,
-          max: 4000,
-          step: 50,
-          markList: [
-            {
-              label: '100',
-              value: 100
-            },
-            {
-              label: '4000',
-              value: 4000
-            }
-          ],
          showTargetInApp: false,
          showTargetInPlugin: false,
          connected: false
@@ -649,89 +614,6 @@ function datasetTemplate({ formData, maxToken }: Props): ModuleItemType[] {
          targets: []
        }
      ]
-    },
-    {
-      moduleId: 'vuc92c',
-      name: 'core.module.template.cfr',
-      avatar: '/imgs/module/cfr.svg',
-      flowType: 'cfr',
-      showStatus: true,
-      position: {
-        x: 758.2985382279098,
-        y: 1124.6527309337314
-      },
-      inputs: [
-        {
-          key: 'switch',
-          type: 'target',
-          label: 'core.module.input.label.switch',
-          valueType: 'any',
-          showTargetInApp: true,
-          showTargetInPlugin: true,
-          connected: false
-        },
-        {
-          key: 'model',
-          type: 'selectExtractModel',
-          label: 'core.module.input.label.aiModel',
-          required: true,
-          valueType: 'string',
-          value: getLLMModel().model,
-          showTargetInApp: false,
-          showTargetInPlugin: false,
-          connected: false
-        },
-        {
-          key: 'systemPrompt',
-          type: 'textarea',
-          label: 'core.module.input.label.cfr background',
-          max: 300,
-          value: formData.cfr.background,
-          valueType: 'string',
-          description: 'core.module.input.description.cfr background',
-          placeholder: 'core.module.input.placeholder.cfr background',
-          showTargetInApp: true,
-          showTargetInPlugin: true,
-          connected: false
-        },
-        {
-          key: 'history',
-          type: 'numberInput',
-          label: 'core.module.input.label.chat history',
-          required: true,
-          min: 0,
-          max: 30,
-          valueType: 'chatHistory',
-          value: 6,
-          showTargetInApp: true,
-          showTargetInPlugin: true,
-          connected: false
-        },
-        {
-          key: 'userChatInput',
-          type: 'target',
-          label: 'core.module.input.label.user question',
-          required: true,
-          valueType: 'string',
-          showTargetInApp: true,
-          showTargetInPlugin: true,
-          connected: true
-        }
-      ],
-      outputs: [
-        {
-          key: 'system_text',
-          label: 'core.module.output.label.cfr result',
-          valueType: 'string',
-          type: 'source',
-          targets: [
-            {
-              moduleId: 'datasetSearch',
-              key: 'userChatInput'
-            }
-          ]
-        }
-      ]
    }
  ];

--- a/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts
+++ b/projects/app/src/pages/api/core/app/form2Modules/fastgpt-universal.ts
@@ -290,7 +290,7 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
          valueType: 'string',
          targets: [
            {
-              moduleId: 'vuc92c',
+              moduleId: 'datasetSearch',
              key: 'userChatInput'
            }
          ]
@@ -335,19 +335,6 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
          label: '最低相关性',
          value: formData.dataset.similarity,
          valueType: 'number',
-          min: 0,
-          max: 1,
-          step: 0.01,
-          markList: [
-            {
-              label: '0',
-              value: 0
-            },
-            {
-              label: '1',
-              value: 1
-            }
-          ],
          showTargetInApp: false,
          showTargetInPlugin: false,
          connected: false
@@ -384,12 +371,33 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
          connected: false
        },
        {
-          key: 'datasetParamsModal',
-          type: 'selectDatasetParamsModal',
+          key: 'datasetSearchUsingExtensionQuery',
+          type: 'hidden',
          label: '',
-          valueType: 'any',
+          valueType: 'boolean',
          showTargetInApp: false,
          showTargetInPlugin: false,
+          value: formData.dataset.datasetSearchUsingExtensionQuery,
+          connected: false
+        },
+        {
+          key: 'datasetSearchExtensionBg',
+          type: 'hidden',
+          label: '',
+          valueType: 'string',
+          showTargetInApp: false,
+          showTargetInPlugin: false,
+          value: formData.dataset.datasetSearchExtensionBg,
+          connected: false
+        },
+        {
+          key: 'datasetSearchExtensionModel',
+          type: 'hidden',
+          label: '',
+          valueType: 'string',
+          showTargetInApp: false,
+          showTargetInPlugin: false,
+          value: formData.dataset.datasetSearchExtensionModel,
          connected: false
        },
        {
@@ -659,89 +667,6 @@ function datasetTemplate(formData: AppSimpleEditFormType): ModuleItemType[] {
          targets: []
        }
      ]
-    },
-    {
-      moduleId: 'vuc92c',
-      name: 'core.module.template.cfr',
-      avatar: '/imgs/module/cfr.svg',
-      flowType: 'cfr',
-      showStatus: true,
-      position: {
-        x: 758.2985382279098,
-        y: 1124.6527309337314
-      },
-      inputs: [
-        {
-          key: 'switch',
-          type: 'target',
-          label: 'core.module.input.label.switch',
-          valueType: 'any',
-          showTargetInApp: true,
-          showTargetInPlugin: true,
-          connected: false
-        },
-        {
-          key: 'model',
-          type: 'selectExtractModel',
-          label: 'core.module.input.label.aiModel',
-          required: true,
-          valueType: 'string',
-          value: getLLMModel().model,
-          showTargetInApp: false,
-          showTargetInPlugin: false,
-          connected: false
-        },
-        {
-          key: 'systemPrompt',
-          type: 'textarea',
-          label: 'core.module.input.label.cfr background',
-          max: 300,
-          value: formData.cfr.background,
-          valueType: 'string',
-          description: 'core.module.input.description.cfr background',
-          placeholder: 'core.module.input.placeholder.cfr background',
-          showTargetInApp: true,
-          showTargetInPlugin: true,
-          connected: false
-        },
-        {
-          key: 'history',
-          type: 'numberInput',
-          label: 'core.module.input.label.chat history',
-          required: true,
-          min: 0,
-          max: 30,
-          valueType: 'chatHistory',
-          value: 6,
-          showTargetInApp: true,
-          showTargetInPlugin: true,
-          connected: false
-        },
-        {
-          key: 'userChatInput',
-          type: 'target',
-          label: 'core.module.input.label.user question',
-          required: true,
-          valueType: 'string',
-          showTargetInApp: true,
-          showTargetInPlugin: true,
-          connected: true
-        }
-      ],
-      outputs: [
-        {
-          key: 'system_text',
-          label: 'core.module.output.label.cfr result',
-          valueType: 'string',
-          type: 'source',
-          targets: [
-            {
-              moduleId: 'datasetSearch',
-              key: 'userChatInput'
-            }
-          ]
-        }
-      ]
    }
  ];

--- a/projects/app/src/pages/api/core/dataset/searchTest.ts
+++ b/projects/app/src/pages/api/core/dataset/searchTest.ts
@@ -9,7 +9,9 @@ import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push';
 import { searchDatasetData } from '@/service/core/dataset/data/controller';
 import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
 import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants';
-import { searchQueryExtension } from '@fastgpt/service/core/ai/functions/queryExtension';
+import { getLLMModel } from '@/service/core/ai/model';
+import { queryExtension } from '@fastgpt/service/core/ai/functions/queryExtension';
+import { datasetSearchQueryExtension } from '@fastgpt/service/core/dataset/search/utils';

 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
@@ -20,13 +22,16 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      limit = 1500,
      similarity,
      searchMode,
-      usingReRank
+      usingReRank,
+
+      datasetSearchUsingExtensionQuery = false,
+      datasetSearchExtensionModel,
+      datasetSearchExtensionBg = ''
    } = req.body as SearchTestProps;

    if (!datasetId || !text) {
      throw new Error('缺少参数');
    }
-
    const start = Date.now();

    // auth dataset role
@@ -37,20 +42,24 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      datasetId,
      per: 'r'
    });
-
    // auth balance
    await authTeamBalance(teamId);

    // query extension
-    // const { queries } = await searchQueryExtension({
-    //   query: text,
-    //   model: global.llmModel[0].model
-    // });
+    const extensionModel =
+      datasetSearchUsingExtensionQuery && datasetSearchExtensionModel
+        ? getLLMModel(datasetSearchExtensionModel)
+        : undefined;
+    const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
+      query: text,
+      extensionModel,
+      extensionBg: datasetSearchExtensionBg
+    });

    const { searchRes, charsLength, ...result } = await searchDatasetData({
      teamId,
-      rawQuery: text,
-      queries: [text],
+      reRankQuery: rewriteQuery,
+      queries: concatQueries,
      model: dataset.vectorModel,
      limit: Math.min(limit, 20000),
      similarity,
@@ -65,7 +74,14 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
      tmbId,
      charsLength,
      model: dataset.vectorModel,
-      source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt
+      source: apikey ? BillSourceEnum.api : BillSourceEnum.fastgpt,
+
+      ...(aiExtensionResult &&
+        extensionModel && {
+          extensionModel: extensionModel.name,
+          extensionInputTokens: aiExtensionResult.inputTokens,
+          extensionOutputTokens: aiExtensionResult.outputTokens
+        })
    });
    if (apikey) {
      updateApiKeyUsage({