4.8.21 feature (#3742)

* model config

* feat: normalization embedding

* adapt unstrea  reasoning response

* remove select app

* perf: dataset search code

* fix: multiple audio video show

* perf: query extension output

* perf: link check

* perf: faq doc

* fix: ts

* feat: support reasoning text output

* feat: workflow support reasoning output

projects/app/src/pages/_error.tsx

@@ -28,12 +28,15 @@ function Error() {
   return (
     <Box whiteSpace={'pre-wrap'}>
       {`出现未捕获的异常。
-1. 私有部署用户，90%由于配置文件不正确/模型未启用导致。请确保系统内每个系列模型至少有一个可用。
+1. 私有部署用户，90%是由于模型配置不正确/模型未启用导致。。
 2. 部分系统不兼容相关API。大部分是苹果的safari 浏览器导致，可以尝试更换 chrome。 
 3. 请关闭浏览器翻译功能，部分翻译导致页面崩溃。
 
 排除3后，打开控制台的 console 查看具体报错信息。
-如果提示 xxx undefined 的话，就是配置文件有错误，或者是缺少可用模型。
+如果提示 xxx undefined 的话，就是模型配置不正确，检查：
+1. 请确保系统内每个系列模型至少有一个可用，可以在【账号-模型提供商】中检查。
+2. 请确保至少有一个知识库文件处理模型（语言模型中有一个开关），否则知识库创建会报错。
+2. 检查模型中一些“对象”参数是否异常（数组和对象），如果为空，可以尝试给个空数组或空对象。
 `}
     </Box>
   );

projects/app/src/pages/api/core/dataset/searchTest.ts

@@ -1,12 +1,12 @@
-import type { NextApiRequest } from 'next';
-import type { SearchTestProps } from '@/global/core/dataset/api.d';
+import type { SearchTestProps, SearchTestResponse } from '@/global/core/dataset/api.d';
 import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
 import { pushGenerateVectorUsage } from '@/service/support/wallet/usage/push';
-import { searchDatasetData } from '@fastgpt/service/core/dataset/search/controller';
+import {
+  deepRagSearch,
+  defaultSearchDatasetData
+} from '@fastgpt/service/core/dataset/search/controller';
 import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
 import { UsageSourceEnum } from '@fastgpt/global/support/wallet/usage/constants';
-import { getLLMModel } from '@fastgpt/service/core/ai/model';
-import { datasetSearchQueryExtension } from '@fastgpt/service/core/dataset/search/utils';
 import {
   checkTeamAIPoints,
   checkTeamReRankPermission
@@ -15,9 +15,9 @@ import { NextAPI } from '@/service/middleware/entry';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { useIPFrequencyLimit } from '@fastgpt/service/common/middle/reqFrequencyLimit';
-import { agentSearchDatasetData } from '@fastgpt/service/core/dataset/search/agent';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
 
-async function handler(req: NextApiRequest) {
+async function handler(req: ApiRequestProps<SearchTestProps>): Promise<SearchTestResponse> {
   const {
     datasetId,
     text,
@@ -26,10 +26,15 @@ async function handler(req: NextApiRequest) {
     searchMode,
     usingReRank,
 
-    datasetSearchUsingExtensionQuery = true,
+    datasetSearchUsingExtensionQuery = false,
     datasetSearchExtensionModel,
-    datasetSearchExtensionBg = ''
-  } = req.body as SearchTestProps;
+    datasetSearchExtensionBg,
+
+    datasetDeepSearch = false,
+    datasetDeepSearchModel,
+    datasetDeepSearchMaxTimes,
+    datasetDeepSearchBg
+  } = req.body;
 
   if (!datasetId || !text) {
     return Promise.reject(CommonErrEnum.missingParams);
@@ -48,29 +53,31 @@ async function handler(req: NextApiRequest) {
   // auth balance
   await checkTeamAIPoints(teamId);
 
-  // query extension
-  const extensionModel =
-    datasetSearchUsingExtensionQuery && datasetSearchExtensionModel
-      ? getLLMModel(datasetSearchExtensionModel)
-      : undefined;
-  const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
-    query: text,
-    extensionModel,
-    extensionBg: datasetSearchExtensionBg
-  });
-
-  const { searchRes, tokens, ...result } = await searchDatasetData({
+  const searchData = {
     histories: [],
     teamId,
-    reRankQuery: rewriteQuery,
-    queries: concatQueries,
+    reRankQuery: text,
+    queries: [text],
     model: dataset.vectorModel,
     limit: Math.min(limit, 20000),
     similarity,
     datasetIds: [datasetId],
     searchMode,
     usingReRank: usingReRank && (await checkTeamReRankPermission(teamId))
-  });
+  };
+  const { searchRes, tokens, queryExtensionResult, deepSearchResult, ...result } = datasetDeepSearch
+    ? await deepRagSearch({
+        ...searchData,
+        datasetDeepSearchModel,
+        datasetDeepSearchMaxTimes,
+        datasetDeepSearchBg
+      })
+    : await defaultSearchDatasetData({
+        ...searchData,
+        datasetSearchUsingExtensionQuery,
+        datasetSearchExtensionModel,
+        datasetSearchExtensionBg
+      });
 
   // push bill
   const { totalPoints } = pushGenerateVectorUsage({
@@ -80,12 +87,16 @@ async function handler(req: NextApiRequest) {
     model: dataset.vectorModel,
     source: apikey ? UsageSourceEnum.api : UsageSourceEnum.fastgpt,
 
-    ...(aiExtensionResult &&
-      extensionModel && {
-        extensionModel: extensionModel.name,
-        extensionInputTokens: aiExtensionResult.inputTokens,
-        extensionOutputTokens: aiExtensionResult.outputTokens
-      })
+    ...(queryExtensionResult && {
+      extensionModel: queryExtensionResult.model,
+      extensionInputTokens: queryExtensionResult.inputTokens,
+      extensionOutputTokens: queryExtensionResult.outputTokens
+    }),
+    ...(deepSearchResult && {
+      deepSearchModel: deepSearchResult.model,
+      deepSearchInputTokens: deepSearchResult.inputTokens,
+      deepSearchOutputTokens: deepSearchResult.outputTokens
+    })
   });
   if (apikey) {
     updateApiKeyUsage({
@@ -97,7 +108,7 @@ async function handler(req: NextApiRequest) {
   return {
     list: searchRes,
     duration: `${((Date.now() - start) / 1000).toFixed(3)}s`,
-    queryExtensionModel: aiExtensionResult?.model,
+    queryExtensionModel: queryExtensionResult?.model,
     ...result
   };
 }

projects/app/src/pages/login/index.tsx

@@ -81,7 +81,7 @@ const Login = ({ ChineseRedirectUrl }: { ChineseRedirectUrl: string }) => {
         router.push(navigateTo);
       }, 300);
     },
-    [lastRoute, router, setUserInfo]
+    [lastRoute, router, setUserInfo, llmModelList]
   );
 
   const DynamicComponent = useMemo(() => {