hiden dataset source (#4152)

* hiden dataset source

* perf: reader

.github/workflows/fastgpt-preview-image.yml

@@ -1,6 +1,9 @@
 name: Preview FastGPT images
 on:
   pull_request_target:
+    paths:
+      - 'projects/app/**'
+      - 'packages/**'
   workflow_dispatch:
 
 jobs:

.prettierignore

@@ -7,4 +7,3 @@ docSite/
 
 pnpm-lock.yaml
 cl100l_base.ts
-dict.json

.vscode/i18n-ally-custom-framework.yml

@@ -17,8 +17,15 @@ usageMatchRegex:
   # you can ignore it and use your own matching rules as well
   - "[^\\w\\d]t\\(['\"`]({key})['\"`]"
   - "[^\\w\\d]commonT\\(['\"`]({key})['\"`]"
+  # 支持 appT("your.i18n.keys")
+  - "[^\\w\\d]appT\\(['\"`]({key})['\"`]"
+  # 支持 datasetT("your.i18n.keys")
+  - "[^\\w\\d]datasetT\\(['\"`]({key})['\"`]"
   - "[^\\w\\d]fileT\\(['\"`]({key})['\"`]"
+  - "[^\\w\\d]publishT\\(['\"`]({key})['\"`]"
   - "[^\\w\\d]workflowT\\(['\"`]({key})['\"`]"
+  - "[^\\w\\d]userT\\(['\"`]({key})['\"`]"
+  - "[^\\w\\d]chatT\\(['\"`]({key})['\"`]"
   - "[^\\w\\d]i18nT\\(['\"`]({key})['\"`]"
 
 # A RegEx to set a custom scope range. This scope will be used as a prefix when detecting keys

deploy/docker/docker-compose-milvus.yml

@@ -114,15 +114,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.1 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.0 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.0 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.1 # git
+    image: ghcr.io/labring/fastgpt:v4.9.0 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.0 # 阿里云
     ports:
       - 3000:3000
     networks:
@@ -175,8 +175,7 @@ services:
 
   # AI Proxy
   aiproxy:
-    image: ghcr.io/labring/aiproxy:v0.1.3
+    image: 'ghcr.io/labring/aiproxy:latest'
-    # image: registry.cn-hangzhou.aliyuncs.com/labring/aiproxy:v0.1.3 # 阿里云
     container_name: aiproxy
     restart: unless-stopped
     depends_on:

deploy/docker/docker-compose-pgvector.yml

@@ -72,15 +72,15 @@ services:
   # fastgpt
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.1 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.0 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.0 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.1 # git
+    image: ghcr.io/labring/fastgpt:v4.9.0 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.0 # 阿里云
     ports:
       - 3000:3000
     networks:
@@ -132,8 +132,7 @@ services:
 
   # AI Proxy
   aiproxy:
-    image: ghcr.io/labring/aiproxy:v0.1.3
+    image: 'ghcr.io/labring/aiproxy:latest'
-    # image: registry.cn-hangzhou.aliyuncs.com/labring/aiproxy:v0.1.3 # 阿里云
     container_name: aiproxy
     restart: unless-stopped
     depends_on:

deploy/docker/docker-compose-zilliz.yml

@@ -53,15 +53,15 @@ services:
         wait $$!
   sandbox:
     container_name: sandbox
-    image: ghcr.io/labring/fastgpt-sandbox:v4.9.1 # git
+    image: ghcr.io/labring/fastgpt-sandbox:v4.9.0 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.1 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt-sandbox:v4.9.0 # 阿里云
     networks:
       - fastgpt
     restart: always
   fastgpt:
     container_name: fastgpt
-    image: ghcr.io/labring/fastgpt:v4.9.1 # git
+    image: ghcr.io/labring/fastgpt:v4.9.0 # git
-    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.1 # 阿里云
+    # image: registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.9.0 # 阿里云
     ports:
       - 3000:3000
     networks:
@@ -113,8 +113,7 @@ services:
 
   # AI Proxy
   aiproxy:
-    image: ghcr.io/labring/aiproxy:v0.1.3
+    image: 'ghcr.io/labring/aiproxy:latest'
-    # image: registry.cn-hangzhou.aliyuncs.com/labring/aiproxy:v0.1.3 # 阿里云
     container_name: aiproxy
     restart: unless-stopped
     depends_on:

docSite/content/zh-cn/docs/development/docker.md

@@ -56,7 +56,7 @@ weight: 707
 
 ### zilliz cloud版本
 
-Zilliz Cloud 由 Milvus 原厂打造，是全托管的 SaaS 向量数据库服务，性能优于 Milvus 并提供 SLA，点击使用 [Zilliz Cloud](https://zilliz.com.cn/)。
+Milvus 的全托管服务，性能优于 Milvus 并提供 SLA，点击使用 [Zilliz Cloud](https://zilliz.com.cn/)。
 
 由于向量库使用了 Cloud，无需占用本地资源，无需太关注。
 

docSite/content/zh-cn/docs/development/upgrading/491.md

@@ -7,39 +7,12 @@ toc: true
 weight: 799
 ---
 
-## 更新指南
-
-### 1. 做好数据库备份
-
-### 2. 更新镜像
-
-- 更新 FastGPT 镜像 tag: v4.9.1
-- 更新 FastGPT 商业版镜像 tag: v4.9.1
-- Sandbox 镜像，可以不更新
-- AIProxy 镜像修改为: registry.cn-hangzhou.aliyuncs.com/labring/aiproxy:v0.1.3
-
-### 3. 执行升级脚本
-
-从任意终端，发起 1 个 HTTP 请求。其中 {{rootkey}} 替换成环境变量里的 `rootkey`；{{host}} 替换成**FastGPT 域名**。
-
-```bash
-curl --location --request POST 'https://{{host}}/api/admin/initv491' \
---header 'rootkey: {{rootkey}}' \
---header 'Content-Type: application/json'
-```
-
-**脚本功能**
-
-重新使用最新的 jieba 分词库进行分词处理。时间较长，可以从日志里查看进度。
-
 ## 🚀 新增内容
 
 1. 商业版支持单团队模式，更好的管理内部成员。
 2. 知识库分块阅读器。
 3. API 知识库支持 PDF 增强解析。
 4. 邀请团队成员，改为邀请链接模式。
-5. 支持混合检索权重设置。
-6. 支持重排模型选择和权重设置，同时调整了知识库搜索权重计算方式，改成 搜索权重 + 重排权重，而不是向量检索权重+全文检索权重+重排权重。
 
 ## ⚙️ 优化
 
@@ -48,7 +21,6 @@ curl --location --request POST 'https://{{host}}/api/admin/initv491' \
 3. 增加依赖包安全版本检测，并升级部分依赖包。
 4. 模型测试代码。
 5. 优化思考过程解析逻辑：只要配置了模型支持思考，均会解析 <think> 标签，不会因为对话时，关闭思考而不解析。
-6. 载入最新 jieba 分词库，增强全文检索分词效果。
 
 ## 🐛 修复
 
@@ -60,5 +32,3 @@ curl --location --request POST 'https://{{host}}/api/admin/initv491' \
 6. 模型渠道测试时，实际未指定渠道测试。
 7. 新增自定义模型时，会把默认模型字段也保存，导致默认模型误判。
 8. 修复 promp 模式工具调用，未判空思考链，导致 UI 错误展示。
-9. 编辑应用信息导致头像丢失。
-10. 分享链接标题会被刷新掉。

package.json

@@ -12,7 +12,7 @@
     "previewIcon": "node ./scripts/icon/index.js",
     "api:gen": "tsc ./scripts/openapi/index.ts && node ./scripts/openapi/index.js && npx @redocly/cli build-docs ./scripts/openapi/openapi.json -o ./projects/app/public/openapi/index.html",
     "create:i18n": "node ./scripts/i18n/index.js",
-    "test": "vitest run --exclude 'test/cases/spec'",
+    "test": "vitest run --exclude './projects/app/src/test/**'",
     "test:all": "vitest run",
     "test:workflow": "vitest run workflow"
   },
@@ -20,9 +20,9 @@
     "@chakra-ui/cli": "^2.4.1",
     "@vitest/coverage-v8": "^3.0.2",
     "husky": "^8.0.3",
-    "i18next": "23.16.8",
+    "i18next": "23.11.5",
     "lint-staged": "^13.3.0",
-    "next-i18next": "15.4.2",
+    "next-i18next": "15.3.0",
     "prettier": "3.2.4",
     "react-i18next": "14.1.2",
     "vitest": "^3.0.2",

packages/global/common/system/types/index.d.ts

@@ -6,7 +6,7 @@ import type {
   EmbeddingModelItemType,
   AudioSpeechModels,
   STTModelType,
-  RerankModelItemType
+  ReRankModelItemType
 } from '../../../core/ai/model.d';
 import { SubTypeEnum } from '../../../support/wallet/sub/constants';
 
@@ -35,7 +35,7 @@ export type FastGPTConfigFileType = {
   // Abandon
   llmModels?: ChatModelItemType[];
   vectorModels?: EmbeddingModelItemType[];
-  reRankModels?: RerankModelItemType[];
+  reRankModels?: ReRankModelItemType[];
   audioSpeechModels?: TTSModelType[];
   whisperModel?: STTModelType;
 };

packages/global/core/ai/model.d.ts

@@ -72,7 +72,7 @@ export type EmbeddingModelItemType = PriceType &
     queryConfig?: Record<string, any>; // Custom parameters for query
   };
 
-export type RerankModelItemType = PriceType &
+export type ReRankModelItemType = PriceType &
   BaseModelItemType & {
     type: ModelTypeEnum.rerank;
   };

packages/global/core/app/type.d.ts

@@ -71,20 +71,6 @@ export type AppDetailType = AppSchema & {
   permission: AppPermission;
 };
 
-export type AppDatasetSearchParamsType = {
-  searchMode: `${DatasetSearchModeEnum}`;
-  limit?: number; // limit max tokens
-  similarity?: number;
-  embeddingWeight?: number; // embedding weight, fullText weight = 1 - embeddingWeight
-
-  usingReRank?: boolean;
-  rerankModel?: string;
-  rerankWeight?: number;
-
-  datasetSearchUsingExtensionQuery?: boolean;
-  datasetSearchExtensionModel?: string;
-  datasetSearchExtensionBg?: string;
-};
 export type AppSimpleEditFormType = {
   // templateId: string;
   aiSettings: {
@@ -102,7 +88,14 @@ export type AppSimpleEditFormType = {
   };
   dataset: {
     datasets: SelectedDatasetType;
-  } & AppDatasetSearchParamsType;
+    searchMode: `${DatasetSearchModeEnum}`;
+    similarity?: number;
+    limit?: number;
+    usingReRank?: boolean;
+    datasetSearchUsingExtensionQuery?: boolean;
+    datasetSearchExtensionModel?: string;
+    datasetSearchExtensionBg?: string;
+  };
   selectedTools: FlowNodeTemplateType[];
   chatConfig: AppChatConfigType;
 };

packages/global/core/app/utils.ts

@@ -24,11 +24,9 @@ export const getDefaultAppForm = (): AppSimpleEditFormType => {
     dataset: {
       datasets: [],
       similarity: 0.4,
-      limit: 3000,
+      limit: 1500,
       searchMode: DatasetSearchModeEnum.embedding,
       usingReRank: false,
-      rerankModel: '',
-      rerankWeight: 0.5,
       datasetSearchUsingExtensionQuery: true,
       datasetSearchExtensionBg: ''
     },
@@ -108,24 +106,10 @@ export const appWorkflow2Form = ({
       defaultAppForm.dataset.searchMode =
         findInputValueByKey(node.inputs, NodeInputKeyEnum.datasetSearchMode) ||
         DatasetSearchModeEnum.embedding;
-      defaultAppForm.dataset.embeddingWeight = findInputValueByKey(
-        node.inputs,
-        NodeInputKeyEnum.datasetSearchEmbeddingWeight
-      );
-      // Rerank
       defaultAppForm.dataset.usingReRank = !!findInputValueByKey(
         node.inputs,
         NodeInputKeyEnum.datasetSearchUsingReRank
       );
-      defaultAppForm.dataset.rerankModel = findInputValueByKey(
-        node.inputs,
-        NodeInputKeyEnum.datasetSearchRerankModel
-      );
-      defaultAppForm.dataset.rerankWeight = findInputValueByKey(
-        node.inputs,
-        NodeInputKeyEnum.datasetSearchRerankWeight
-      );
-      // Query extension
       defaultAppForm.dataset.datasetSearchUsingExtensionQuery = findInputValueByKey(
         node.inputs,
         NodeInputKeyEnum.datasetSearchUsingExtensionQuery

packages/global/core/dataset/constants.ts

@@ -185,7 +185,7 @@ export enum SearchScoreTypeEnum {
 }
 export const SearchScoreTypeMap = {
   [SearchScoreTypeEnum.embedding]: {
-    label: i18nT('common:core.dataset.search.mode.embedding'),
+    label: i18nT('common:core.dataset.search.score.embedding'),
     desc: i18nT('common:core.dataset.search.score.embedding desc'),
     showScore: true
   },

packages/global/core/workflow/constants.ts

@@ -154,12 +154,7 @@ export enum NodeInputKeyEnum {
   datasetSimilarity = 'similarity',
   datasetMaxTokens = 'limit',
   datasetSearchMode = 'searchMode',
-  datasetSearchEmbeddingWeight = 'embeddingWeight',
-
   datasetSearchUsingReRank = 'usingReRank',
-  datasetSearchRerankWeight = 'rerankWeight',
-  datasetSearchRerankModel = 'rerankModel',
-
   datasetSearchUsingExtensionQuery = 'datasetSearchUsingExtensionQuery',
   datasetSearchExtensionModel = 'datasetSearchExtensionModel',
   datasetSearchExtensionBg = 'datasetSearchExtensionBg',

packages/global/core/workflow/runtime/type.d.ts

@@ -133,9 +133,6 @@ export type DispatchNodeResponseType = {
   similarity?: number;
   limit?: number;
   searchMode?: `${DatasetSearchModeEnum}`;
-  embeddingWeight?: number;
-  rerankModel?: string;
-  rerankWeight?: number;
   searchUsingReRank?: boolean;
   queryExtensionResult?: {
     model: string;

packages/global/core/workflow/template/system/datasetSearch.ts

@@ -64,14 +64,6 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
       valueType: WorkflowIOValueTypeEnum.string,
       value: DatasetSearchModeEnum.embedding
     },
-    {
-      key: NodeInputKeyEnum.datasetSearchEmbeddingWeight,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      label: '',
-      valueType: WorkflowIOValueTypeEnum.number,
-      value: 0.5
-    },
-    // Rerank
     {
       key: NodeInputKeyEnum.datasetSearchUsingReRank,
       renderTypeList: [FlowNodeInputTypeEnum.hidden],
@@ -79,20 +71,6 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
       valueType: WorkflowIOValueTypeEnum.boolean,
       value: false
     },
-    {
-      key: NodeInputKeyEnum.datasetSearchRerankModel,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      label: '',
-      valueType: WorkflowIOValueTypeEnum.string
-    },
-    {
-      key: NodeInputKeyEnum.datasetSearchRerankWeight,
-      renderTypeList: [FlowNodeInputTypeEnum.hidden],
-      label: '',
-      valueType: WorkflowIOValueTypeEnum.number,
-      value: 0.5
-    },
-    // Query Extension
     {
       key: NodeInputKeyEnum.datasetSearchUsingExtensionQuery,
       renderTypeList: [FlowNodeInputTypeEnum.hidden],
@@ -113,7 +91,6 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
       valueType: WorkflowIOValueTypeEnum.string,
       value: ''
     },
-
     {
       key: NodeInputKeyEnum.authTmbId,
       renderTypeList: [FlowNodeInputTypeEnum.hidden],

packages/global/package.json

@@ -10,7 +10,7 @@
     "js-yaml": "^4.1.0",
     "jschardet": "3.1.1",
     "nanoid": "^5.1.3",
-    "next": "14.2.24",
+    "next": "14.2.21",
     "openai": "4.61.0",
     "openapi-types": "^12.1.3",
     "json5": "^2.2.3",

packages/service/common/file/image/controller.ts

@@ -76,7 +76,7 @@ export const refreshSourceAvatar = async (
   const newId = getIdFromPath(path);
   const oldId = getIdFromPath(oldPath);
 
-  if (!newId || newId === oldId) return;
+  if (!newId) return;
 
   await MongoImage.updateOne({ _id: newId }, { $unset: { expiredTime: 1 } }, { session });
 

packages/service/common/string/jieba.ts

@@ -1,13 +1,4 @@
-import { Jieba } from '@node-rs/jieba';
+import { cut } from '@node-rs/jieba';
-
-let jieba: Jieba | undefined;
-
-(async () => {
-  const dictData = await import('./dict.json');
-  // @ts-ignore
-  const dictBuffer = Buffer.from(dictData.dict?.replace(/\\n/g, '\n'), 'utf-8');
-  jieba = Jieba.withDict(dictBuffer);
-})();
 
 const stopWords = new Set([
   '--',
@@ -1518,10 +1509,8 @@ const stopWords = new Set([
   ]
 ]);
 
-export async function jiebaSplit({ text }: { text: string }) {
+export function jiebaSplit({ text }: { text: string }) {
-  text = text.replace(/[#*`_~>[\](){}|]/g, '').replace(/\S*https?\S*/gi, '');
+  const tokens = cut(text, true);
-
-  const tokens = (await jieba!.cutAsync(text, true)) as string[];
 
   return (
     tokens

packages/service/common/system/utils.ts

@@ -30,8 +30,6 @@ export const isInternalAddress = (url: string): boolean => {
       return true;
     }
 
-    if (process.env.CHECK_INTERNAL_IP !== 'true') return false;
-
     // For IP addresses, check if they are internal
     const ipv4Pattern = /^(\d{1,3}\.){3}\d{1,3}$/;
     if (!ipv4Pattern.test(hostname)) {

packages/service/core/ai/config/utils.ts

@@ -8,7 +8,7 @@ import {
   EmbeddingModelItemType,
   TTSModelType,
   STTModelType,
-  RerankModelItemType
+  ReRankModelItemType
 } from '@fastgpt/global/core/ai/model.d';
 import { debounce } from 'lodash';
 import {
@@ -94,7 +94,7 @@ export const loadSystemModels = async (init = false) => {
   global.embeddingModelMap = new Map<string, EmbeddingModelItemType>();
   global.ttsModelMap = new Map<string, TTSModelType>();
   global.sttModelMap = new Map<string, STTModelType>();
-  global.reRankModelMap = new Map<string, RerankModelItemType>();
+  global.reRankModelMap = new Map<string, ReRankModelItemType>();
   // @ts-ignore
   global.systemDefaultModel = {};
 

packages/service/core/ai/model.ts

@@ -38,7 +38,7 @@ export function getSTTModel(model?: string) {
 }
 
 export const getDefaultRerankModel = () => global?.systemDefaultModel.rerank!;
-export function getRerankModel(model?: string) {
+export function getReRankModel(model?: string) {
   if (!model) return getDefaultRerankModel();
   return global.reRankModelMap.get(model) || getDefaultRerankModel();
 }

packages/service/core/ai/rerank/index.ts

@@ -2,7 +2,7 @@ import { addLog } from '../../../common/system/log';
 import { POST } from '../../../common/api/serverRequest';
 import { getDefaultRerankModel } from '../model';
 import { getAxiosConfig } from '../config';
-import { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
+import { ReRankModelItemType } from '@fastgpt/global/core/ai/model.d';
 
 type PostReRankResponse = {
   id: string;
@@ -19,7 +19,7 @@ export function reRankRecall({
   documents,
   headers
 }: {
-  model?: RerankModelItemType;
+  model?: ReRankModelItemType;
   query: string;
   documents: { id: string; text: string }[];
   headers?: Record<string, string>;

packages/service/core/ai/type.d.ts

@@ -1,7 +1,7 @@
 import { ModelTypeEnum } from '@fastgpt/global/core/ai/model';
 import {
   STTModelType,
-  RerankModelItemType,
+  ReRankModelItemType,
   TTSModelType,
   EmbeddingModelItemType,
   LLMModelItemType
@@ -18,7 +18,7 @@ export type SystemModelItemType =
   | EmbeddingModelItemType
   | TTSModelType
   | STTModelType
-  | RerankModelItemType;
+  | ReRankModelItemType;
 
 export type SystemDefaultModelType = {
   [ModelTypeEnum.llm]?: LLMModelItemType;
@@ -28,7 +28,7 @@ export type SystemDefaultModelType = {
   [ModelTypeEnum.embedding]?: EmbeddingModelItemType;
   [ModelTypeEnum.tts]?: TTSModelType;
   [ModelTypeEnum.stt]?: STTModelType;
-  [ModelTypeEnum.rerank]?: RerankModelItemType;
+  [ModelTypeEnum.rerank]?: ReRankModelItemType;
 };
 
 declare global {
@@ -38,7 +38,7 @@ declare global {
   var embeddingModelMap: Map<string, EmbeddingModelItemType>;
   var ttsModelMap: Map<string, TTSModelType>;
   var sttModelMap: Map<string, STTModelType>;
-  var reRankModelMap: Map<string, RerankModelItemType>;
+  var reRankModelMap: Map<string, ReRankModelItemType>;
 
   var systemActiveModelList: SystemModelItemType[];
   var systemDefaultModel: SystemDefaultModelType;

packages/service/core/dataset/data/dataTextSchema.ts

@@ -41,7 +41,7 @@ try {
     }
   );
   DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, collectionId: 1 });
-  DatasetDataTextSchema.index({ dataId: 'hashed' });
+  DatasetDataTextSchema.index({ dataId: 1 }, { unique: true });
 } catch (error) {
   console.log(error);
 }

packages/service/core/dataset/data/schema.ts

@@ -86,8 +86,7 @@ const DatasetDataSchema = new Schema({
 
   // Abandon
   fullTextToken: String,
-  initFullText: Boolean,
+  initFullText: Boolean
-  initJieba: Boolean
 });
 
 try {
@@ -104,9 +103,6 @@ try {
   DatasetDataSchema.index({ updateTime: 1 });
   // rebuild data
   DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
-
-  // 为查询 initJieba 字段不存在的数据添加索引
-  DatasetDataSchema.index({ initJieba: 1, updateTime: 1 });
 } catch (error) {
   console.log(error);
 }

packages/service/core/dataset/search/controller.ts

@@ -16,7 +16,7 @@ import { reRankRecall } from '../../../core/ai/rerank';
 import { countPromptTokens } from '../../../common/string/tiktoken/index';
 import { datasetSearchResultConcat } from '@fastgpt/global/core/dataset/search/utils';
 import { hashStr } from '@fastgpt/global/common/string/tools';
-import { jiebaSplit } from '../../../common/string/jieba/index';
+import { jiebaSplit } from '../../../common/string/jieba';
 import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
 import { Types } from '../../../common/mongo';
 import json5 from 'json5';
@@ -27,7 +27,6 @@ import { ChatItemType } from '@fastgpt/global/core/chat/type';
 import { POST } from '../../../common/api/plusRequest';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { datasetSearchQueryExtension } from './utils';
-import type { RerankModelItemType } from '@fastgpt/global/core/ai/model.d';
 
 export type SearchDatasetDataProps = {
   histories: ChatItemType[];
@@ -40,11 +39,7 @@ export type SearchDatasetDataProps = {
   [NodeInputKeyEnum.datasetSimilarity]?: number; // min distance
   [NodeInputKeyEnum.datasetMaxTokens]: number; // max Token limit
   [NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
-  [NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
-
   [NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
-  [NodeInputKeyEnum.datasetSearchRerankModel]?: RerankModelItemType;
-  [NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
 
   /* 
     {
@@ -80,16 +75,13 @@ export type SearchDatasetDataResponse = {
 };
 
 export const datasetDataReRank = async ({
-  rerankModel,
   data,
   query
 }: {
-  rerankModel?: RerankModelItemType;
   data: SearchDataResponseItemType[];
   query: string;
 }): Promise<SearchDataResponseItemType[]> => {
   const results = await reRankRecall({
-    model: rerankModel,
     query,
     documents: data.map((item) => ({
       id: item.id,
@@ -162,10 +154,7 @@ export async function searchDatasetData(
     similarity = 0,
     limit: maxTokens,
     searchMode = DatasetSearchModeEnum.embedding,
-    embeddingWeight = 0.5,
     usingReRank = false,
-    rerankModel,
-    rerankWeight = 0.5,
     datasetIds = [],
     collectionFilterMatch
   } = props;
@@ -537,7 +526,7 @@ export async function searchDatasetData(
                 $match: {
                   teamId: new Types.ObjectId(teamId),
                   datasetId: new Types.ObjectId(id),
-                  $text: { $search: await jiebaSplit({ text: query }) },
+                  $text: { $search: jiebaSplit({ text: query }) },
                   ...(filterCollectionIdList
                     ? {
                         collectionId: {
@@ -722,7 +711,6 @@ export async function searchDatasetData(
     });
     try {
       return await datasetDataReRank({
-        rerankModel,
         query: reRankQuery,
         data: filterSameDataResults
       });
@@ -733,26 +721,11 @@ export async function searchDatasetData(
   })();
 
   // embedding recall and fullText recall rrf concat
-  const baseK = 120;
+  const rrfConcatResults = datasetSearchResultConcat([
-  const embK = Math.round(baseK * (1 - embeddingWeight)); // 搜索结果的 k 值
+    { k: 60, list: embeddingRecallResults },
-  const fullTextK = Math.round(baseK * embeddingWeight); // rerank 结果的 k 值
+    { k: 60, list: fullTextRecallResults },
-
+    { k: 58, list: reRankResults }
-  const rrfSearchResult = datasetSearchResultConcat([
-    { k: embK, list: embeddingRecallResults },
-    { k: fullTextK, list: fullTextRecallResults }
   ]);
-  const rrfConcatResults = (() => {
-    if (reRankResults.length === 0) return rrfSearchResult;
-    if (rerankWeight === 1) return reRankResults;
-
-    const searchK = Math.round(baseK * rerankWeight); // 搜索结果的 k 值
-    const rerankK = Math.round(baseK * (1 - rerankWeight)); // rerank 结果的 k 值
-
-    return datasetSearchResultConcat([
-      { k: searchK, list: rrfSearchResult },
-      { k: rerankK, list: reRankResults }
-    ]);
-  })();
 
   // remove same q and a data
   set = new Set<string>();

packages/service/core/workflow/dispatch/dataset/search.ts

@@ -6,7 +6,7 @@ import { formatModelChars2Points } from '../../../../support/wallet/usage/utils'
 import type { SelectedDatasetType } from '@fastgpt/global/core/workflow/api.d';
 import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/runtime/type';
-import { getEmbeddingModel, getRerankModel } from '../../../ai/model';
+import { getEmbeddingModel } from '../../../ai/model';
 import { deepRagSearch, defaultSearchDatasetData } from '../../../dataset/search/controller';
 import { NodeInputKeyEnum, NodeOutputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
@@ -22,14 +22,9 @@ type DatasetSearchProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
   [NodeInputKeyEnum.datasetSimilarity]: number;
   [NodeInputKeyEnum.datasetMaxTokens]: number;
-  [NodeInputKeyEnum.userChatInput]?: string;
   [NodeInputKeyEnum.datasetSearchMode]: `${DatasetSearchModeEnum}`;
-  [NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
+  [NodeInputKeyEnum.userChatInput]?: string;
-
   [NodeInputKeyEnum.datasetSearchUsingReRank]: boolean;
-  [NodeInputKeyEnum.datasetSearchRerankModel]?: string;
-  [NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
-
   [NodeInputKeyEnum.collectionFilterMatch]: string;
   [NodeInputKeyEnum.authTmbId]?: boolean;
 
@@ -58,14 +53,11 @@ export async function dispatchDatasetSearch(
       datasets = [],
       similarity,
       limit = 1500,
+      usingReRank,
+      searchMode,
       userChatInput = '',
       authTmbId = false,
       collectionFilterMatch,
-      searchMode,
-      embeddingWeight,
-      usingReRank,
-      rerankModel,
-      rerankWeight,
 
       datasetSearchUsingExtensionQuery,
       datasetSearchExtensionModel,
@@ -130,10 +122,7 @@ export async function dispatchDatasetSearch(
     limit,
     datasetIds,
     searchMode,
-    embeddingWeight,
     usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
-    rerankModel: getRerankModel(rerankModel),
-    rerankWeight,
     collectionFilterMatch
   };
   const {
@@ -230,9 +219,6 @@ export async function dispatchDatasetSearch(
     similarity: usingSimilarityFilter ? similarity : undefined,
     limit,
     searchMode,
-    embeddingWeight: searchMode === DatasetSearchModeEnum.mixedRecall ? embeddingWeight : undefined,
-    rerankModel: usingReRank ? getRerankModel(rerankModel)?.name : undefined,
-    rerankWeight: usingReRank ? rerankWeight : undefined,
     searchUsingReRank: searchUsingReRank,
     quoteList: searchRes,
     queryExtensionResult,

packages/service/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "dependencies": {
     "@fastgpt/global": "workspace:*",
-    "@node-rs/jieba": "2.0.1",
+    "@node-rs/jieba": "1.10.0",
     "@xmldom/xmldom": "^0.8.10",
     "@zilliz/milvus2-sdk-node": "2.4.2",
     "axios": "^1.8.2",
@@ -26,7 +26,7 @@
     "mammoth": "^1.6.0",
     "mongoose": "^8.10.1",
     "multer": "1.4.5-lts.1",
-    "next": "14.2.24",
+    "next": "14.2.21",
     "nextjs-cors": "^2.2.0",
     "node-cron": "^3.0.3",
     "node-xlsx": "^0.24.0",

packages/service/type.d.ts

@@ -1,7 +1,7 @@
 import { FastGPTFeConfigsType, SystemEnvType } from '@fastgpt/global/common/system/types';
 import {
   TTSModelType,
-  RerankModelItemType,
+  ReRankModelItemType,
   STTModelType,
   EmbeddingModelItemType,
   LLMModelItemType

packages/web/common/system/nextjs.ts

@@ -1,4 +1,4 @@
-import { I18nNsType } from '@fastgpt/web/types/i18next';
+import { I18nNsType } from '../../types/i18next';
 import { serverSideTranslations } from 'next-i18next/serverSideTranslations';
 
 export const serviceSideProps = async (content: any, ns: I18nNsType = []) => {
@@ -9,7 +9,7 @@ export const serviceSideProps = async (content: any, ns: I18nNsType = []) => {
   const deviceSize = content.req?.cookies?.NEXT_DEVICE_SIZE || null;
 
   return {
-    ...(await serverSideTranslations(lang, ['common', ...ns], undefined, extraLng)),
+    ...(await serverSideTranslations(lang, ['common', ...ns], null, extraLng)),
     deviceSize
   };
 };

packages/web/components/common/Icon/constants.ts

@@ -212,9 +212,7 @@ export const iconPaths = {
   'core/dataset/manualCollection': () => import('./icons/core/dataset/manualCollection.svg'),
   'core/dataset/mixedRecall': () => import('./icons/core/dataset/mixedRecall.svg'),
   'core/dataset/modeEmbedding': () => import('./icons/core/dataset/modeEmbedding.svg'),
-  'core/dataset/questionExtension': () => import('./icons/core/dataset/questionExtension.svg'),
   'core/dataset/rerank': () => import('./icons/core/dataset/rerank.svg'),
-  'core/dataset/searchfilter': () => import('./icons/core/dataset/searchfilter.svg'),
   'core/dataset/splitLight': () => import('./icons/core/dataset/splitLight.svg'),
   'core/dataset/tableCollection': () => import('./icons/core/dataset/tableCollection.svg'),
   'core/dataset/tag': () => import('./icons/core/dataset/tag.svg'),
@@ -429,7 +427,6 @@ export const iconPaths = {
   'price/bg': () => import('./icons/price/bg.svg'),
   'price/right': () => import('./icons/price/right.svg'),
   save: () => import('./icons/save.svg'),
-  sliderTag: () => import('./icons/sliderTag.svg'),
   stop: () => import('./icons/stop.svg'),
   'support/account/laf': () => import('./icons/support/account/laf.svg'),
   'support/account/loginoutLight': () => import('./icons/support/account/loginoutLight.svg'),

packages/web/components/common/Icon/icons/core/dataset/questionExtension.svg

@@ -1,3 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 17 16" >
-  <path d="M11.3317 4.63043L12.3128 5.61147L8.98815 8.9361L6.74675 6.6947C6.68372 6.63154 6.60886 6.58144 6.52644 6.54725C6.44402 6.51306 6.35567 6.49546 6.26645 6.49546C6.17722 6.49546 6.08887 6.51306 6.00646 6.54725C5.92404 6.58144 5.84918 6.63154 5.78615 6.6947L1.69849 10.7892C1.63542 10.8522 1.58538 10.9271 1.55125 11.0095C1.51711 11.0919 1.49954 11.1803 1.49954 11.2695C1.49954 11.3587 1.51711 11.447 1.55125 11.5294C1.58538 11.6118 1.63542 11.6867 1.69849 11.7498C1.76156 11.8128 1.83644 11.8629 1.91885 11.897C2.00126 11.9312 2.08959 11.9487 2.17879 11.9487C2.26799 11.9487 2.35632 11.9312 2.43873 11.897C2.52114 11.8629 2.59601 11.8128 2.65909 11.7498L6.26304 8.13901L8.50444 10.3804C8.77014 10.6461 9.19934 10.6461 9.46504 10.3804L13.2734 6.57888L14.2544 7.55992C14.302 7.6066 14.3623 7.63828 14.4277 7.65104C14.4932 7.66379 14.5609 7.65706 14.6226 7.63167C14.6842 7.60628 14.7371 7.56336 14.7746 7.50822C14.8121 7.45308 14.8326 7.38815 14.8335 7.32147V4.39198C14.8354 4.34733 14.8281 4.30276 14.812 4.26104C14.796 4.21931 14.7716 4.18132 14.7403 4.1494C14.709 4.11747 14.6716 4.0923 14.6302 4.07544C14.5888 4.05858 14.5444 4.05038 14.4997 4.05135H11.577C11.5097 4.05095 11.4439 4.07047 11.3877 4.10745C11.3315 4.14443 11.2876 4.1972 11.2613 4.25913C11.2351 4.32106 11.2278 4.38937 11.2403 4.45544C11.2529 4.52152 11.2847 4.5824 11.3317 4.63043Z" />
-</svg>

packages/web/components/common/Icon/icons/core/dataset/searchfilter.svg

@@ -1,3 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 17 16" >
-  <path fill-rule="evenodd" clip-rule="evenodd" d="M13.8911 3.61032C15.0171 5.17598 14.8761 7.37117 13.4681 8.77915L12.6427 9.60456C12.3733 9.87396 11.9365 9.87396 11.6671 9.60456L6.97652 4.91394C6.70711 4.64454 6.70711 4.20775 6.97652 3.93834L7.80192 3.11294C9.20207 1.71279 11.3807 1.56556 12.9446 2.67127L13.8718 1.74404C14.1322 1.48369 14.5543 1.48369 14.8146 1.74404C15.075 2.00439 15.075 2.4265 14.8146 2.68685L13.8911 3.61032ZM2.18142 13.4425C1.92107 13.7029 1.92107 14.125 2.18142 14.3853C2.44177 14.6457 2.86388 14.6457 3.12423 14.3853L4.04283 13.4667C5.6079 14.5859 7.79751 14.4428 9.20286 13.0374L10.0283 12.212C10.2977 11.9426 10.2977 11.5058 10.0283 11.2364L9.44649 10.6546L9.94334 10.1578C10.2037 9.89744 10.2037 9.47533 9.94334 9.21498C9.68299 8.95463 9.26088 8.95463 9.00053 9.21498L8.50368 9.71183L6.87902 8.08717L7.37233 7.59386C7.63268 7.33351 7.63268 6.9114 7.37233 6.65105C7.11198 6.3907 6.68987 6.3907 6.42952 6.65105L5.93621 7.14436L5.33765 6.5458C5.06824 6.27639 4.63145 6.27639 4.36205 6.5458L3.53665 7.3712C2.13385 8.774 1.98872 10.9582 3.10124 12.5227L2.18142 13.4425ZM4.84985 7.94362L4.47946 8.31401C3.43548 9.35799 3.43548 11.0506 4.47946 12.0946C5.52344 13.1386 7.21607 13.1386 8.26005 12.0946L8.63044 11.7242L4.84985 7.94362ZM12.5253 7.83635L12.1549 8.20674L8.37433 4.42614L8.74473 4.05575C9.78871 3.01177 11.4813 3.01177 12.5253 4.05575C13.5693 5.09973 13.5693 6.79236 12.5253 7.83635Z" />
-</svg>

packages/web/components/common/Icon/icons/sliderTag.svg

@@ -1,3 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 11 11" >
-  <path d="M5.04123 0.144501L9.47821 4.82132C9.83075 5.19292 10.0273 5.68562 10.0273 6.19784V8.65565C10.0273 9.76022 9.13185 10.6557 8.02728 10.6557H2.05518C0.950606 10.6557 0.0551758 9.76022 0.0551758 8.65565V6.19785C0.0551758 5.68562 0.251705 5.19292 0.604247 4.82132L5.04123 0.144501Z" fill="#3370FF"/>
-</svg>

packages/web/components/common/MyModal/index.tsx

@@ -53,7 +53,6 @@ const MyModal = ({
       allowPinchZoom
       scrollBehavior={'inside'}
       closeOnOverlayClick={closeOnOverlayClick}
-      returnFocusOnClose={false}
     >
       <ModalOverlay />
       <ModalContent

packages/web/components/common/Tabs/LightRowTabs.tsx

@@ -73,11 +73,6 @@ const LightRowTabs = <ValueType = string,>({
               color: activeColor
             }}
             fontWeight={'medium'}
-            onClick={() => {
-              if (value === item.value) return;
-              onChange(item.value);
-            }}
-            {...inlineStyles}
             {...(value === item.value
               ? {
                   color: activeColor,
@@ -87,6 +82,11 @@ const LightRowTabs = <ValueType = string,>({
               : {
                   cursor: 'pointer'
                 })}
+            onClick={() => {
+              if (value === item.value) return;
+              onChange(item.value);
+            }}
+            {...inlineStyles}
           >
             {item.icon && (
               <>

packages/web/hooks/useI18n.ts

@@ -44,7 +44,7 @@ export const useI18nLng = () => {
 
     await i18n?.changeLanguage?.(lang);
 
-    if (!i18n?.hasResourceBundle?.(lang, 'common') && prevLang !== lang) {
+    if (!i18n.hasResourceBundle(lang, 'common') && prevLang !== lang) {
       window?.location?.reload?.();
     }
   };

packages/web/hooks/useLinkedScroll.tsx

@@ -1,4 +1,4 @@
-import { useEffect, useRef, useState, ReactNode } from 'react';
+import { useEffect, useRef, useState, ReactNode, useCallback } from 'react';
 import { LinkedListResponse, LinkedPaginationProps } from '../common/fetch/type';
 import { Box, BoxProps } from '@chakra-ui/react';
 import { useTranslation } from 'next-i18next';
@@ -34,13 +34,13 @@ export function useLinkedScroll<
     bottom: null as { _id: string; index: number } | null
   });
   const containerRef = useRef<HTMLDivElement>(null);
-  const itemRefs = useRef<Map<string, HTMLElement | null>>(new Map());
+  const itemRefs = useRef<(HTMLElement | null)[]>([]);
 
   const scrollToItem = async (id: string, retry = 3) => {
     const itemIndex = dataList.findIndex((item) => item._id === id);
     if (itemIndex === -1) return;
 
-    const element = itemRefs.current.get(id);
+    const element = itemRefs.current[itemIndex];
 
     if (!element || !containerRef.current) {
       if (retry > 0) {
@@ -64,12 +64,11 @@ export function useLinkedScroll<
 
   let scroolSign = useRef(false);
   const { runAsync: loadInitData } = useRequest2(
-    async ({ scrollWhenFinish, refresh } = { scrollWhenFinish: true, refresh: false }) => {
+    async (scrollWhenFinish = true) => {
-      console.log('loadInitData', params);
       if (!currentData || isLoading) return;
 
       const item = dataList.find((item) => item._id === currentData.id);
-      if (item && !refresh) {
+      if (item) {
         scrollToItem(item._id);
         return;
       }

packages/web/i18n/en/chat.json

@@ -49,7 +49,6 @@
   "response.child total points": "Sub-workflow point consumption",
   "response.dataset_concat_length": "Combined total",
   "response.node_inputs": "Node Inputs",
-  "response_hybrid_weight": "Embedding : Full text = {{emb}} : {{text}}",
   "select": "Select",
   "select_file": "Upload File",
   "select_file_img": "Upload file / image",

packages/web/i18n/en/common.json

@@ -107,7 +107,6 @@
   "code_error.user_error.balance_not_enough": "Insufficient Account Balance",
   "code_error.user_error.bin_visitor_guest": "You Are Currently a Guest, Unauthorized to Operate",
   "code_error.user_error.un_auth_user": "User Not Found",
-  "commercial_function_tip": "Please Upgrade to the Commercial Version to Use This Feature: https://doc.fastgpt.cn/docs/commercial/intro/",
   "common.Action": "Action",
   "common.Add": "Add",
   "common.Add New": "Add New",
@@ -257,6 +256,7 @@
   "common.submit_success": "Submitted Successfully",
   "common.submitted": "Submitted",
   "common.support": "Support",
+  "commercial_function_tip": "Please Upgrade to the Commercial Version to Use This Feature: https://doc.fastgpt.cn/docs/commercial/intro/",
   "common.system.Help Chatbot": "Help Chatbot",
   "common.system.Use Helper": "Use Helper",
   "common.ui.textarea.Magnifying": "Magnifying",
@@ -627,7 +627,7 @@
   "core.dataset.search.score.reRank desc": "Calculate the relevance between sentences using the re-rank model, ranging from 0 to 1.",
   "core.dataset.search.score.rrf": "Comprehensive Ranking",
   "core.dataset.search.score.rrf desc": "Merge multiple search results using the reciprocal rank fusion method.",
-  "core.dataset.search.search mode": "Search Method",
+  "core.dataset.search.search mode": "Search Mode",
   "core.dataset.status.active": "Ready",
   "core.dataset.status.syncing": "Syncing",
   "core.dataset.test.Batch test": "Batch Test",
@@ -1025,7 +1025,6 @@
   "question_feedback": "Work order",
   "read_quote": "View citations",
   "required": "Required",
-  "rerank_weight": "Rearrange weights",
   "resume_failed": "Resume Failed",
   "select_reference_variable": "Select Reference Variable",
   "share_link": "Share Link",

packages/web/i18n/zh-CN/chat.json

@@ -49,7 +49,6 @@
   "response.child total points": "子工作流积分消耗",
   "response.dataset_concat_length": "合并后总数",
   "response.node_inputs": "节点输入",
-  "response_hybrid_weight": "语义检索 : 全文检索 = {{emb}} : {{text}}",
   "select": "选择",
   "select_file": "上传文件",
   "select_file_img": "上传文件/图片",

packages/web/i18n/zh-CN/common.json

@@ -111,7 +111,6 @@
   "code_error.user_error.balance_not_enough": "账号余额不足~",
   "code_error.user_error.bin_visitor_guest": "您当前身份为游客，无权操作",
   "code_error.user_error.un_auth_user": "找不到该用户",
-  "commercial_function_tip": "请升级商业版后使用该功能：https://doc.fastgpt.cn/docs/commercial/intro/",
   "common.Action": "操作",
   "common.Add": "添加",
   "common.Add New": "新增",
@@ -261,6 +260,7 @@
   "common.submit_success": "提交成功",
   "common.submitted": "已提交",
   "common.support": "支持",
+  "commercial_function_tip": "请升级商业版后使用该功能：https://doc.fastgpt.cn/docs/commercial/intro/",
   "common.system.Help Chatbot": "机器人助手",
   "common.system.Use Helper": "使用帮助",
   "common.ui.textarea.Magnifying": "放大",
@@ -623,6 +623,7 @@
   "core.dataset.search.mode.fullTextRecall desc": "使用传统的全文检索，适合查找一些关键词和主谓语特殊的数据",
   "core.dataset.search.mode.mixedRecall": "混合检索",
   "core.dataset.search.mode.mixedRecall desc": "使用向量检索与全文检索的综合结果返回，使用 RRF 算法进行排序。",
+  "core.dataset.search.score.embedding": "语义检索",
   "core.dataset.search.score.embedding desc": "通过计算向量之间的距离获取得分，范围为 0~1。",
   "core.dataset.search.score.fullText": "全文检索",
   "core.dataset.search.score.fullText desc": "计算相同关键词的得分，范围为 0~无穷。",
@@ -630,7 +631,7 @@
   "core.dataset.search.score.reRank desc": "通过 Rerank 模型计算句子之间的关联度，范围为 0~1。",
   "core.dataset.search.score.rrf": "综合排名",
   "core.dataset.search.score.rrf desc": "通过倒排计算的方式，合并多个检索结果。",
-  "core.dataset.search.search mode": "搜索方式",
+  "core.dataset.search.search mode": "搜索模式",
   "core.dataset.status.active": "已就绪",
   "core.dataset.status.syncing": "同步中",
   "core.dataset.test.Batch test": "批量测试",
@@ -1028,7 +1029,6 @@
   "question_feedback": "工单咨询",
   "read_quote": "查看引用",
   "required": "必须",
-  "rerank_weight": "重排权重",
   "resume_failed": "恢复失败",
   "select_reference_variable": "选择引用变量",
   "share_link": "分享链接",

packages/web/i18n/zh-Hant/chat.json

@@ -48,7 +48,6 @@
   "response.child total points": "子工作流程點數消耗",
   "response.dataset_concat_length": "合併總數",
   "response.node_inputs": "節點輸入",
-  "response_hybrid_weight": "語義檢索 : 全文檢索 = {{emb}} : {{text}}",
   "select": "選取",
   "select_file": "上傳檔案",
   "select_file_img": "上傳檔案 / 圖片",

packages/web/i18n/zh-Hant/common.json

@@ -106,7 +106,6 @@
   "code_error.user_error.balance_not_enough": "帳戶餘額不足",
   "code_error.user_error.bin_visitor_guest": "您目前身份為訪客，無權操作",
   "code_error.user_error.un_auth_user": "找不到此使用者",
-  "commercial_function_tip": "請升級為商業版後使用此功能：https://doc.fastgpt.cn/docs/commercial/intro/",
   "common.Action": "操作",
   "common.Add": "新增",
   "common.Add New": "新增",
@@ -256,6 +255,7 @@
   "common.submit_success": "送出成功",
   "common.submitted": "已送出",
   "common.support": "支援",
+  "commercial_function_tip": "請升級為商業版後使用此功能：https://doc.fastgpt.cn/docs/commercial/intro/",
   "common.system.Help Chatbot": "機器人助手",
   "common.system.Use Helper": "使用說明",
   "common.ui.textarea.Magnifying": "放大",
@@ -626,7 +626,7 @@
   "core.dataset.search.score.reRank desc": "透過重新排名模型計算句子之間的關聯度，範圍為 0 到 1。",
   "core.dataset.search.score.rrf": "綜合排名",
   "core.dataset.search.score.rrf desc": "使用倒數排名融合方法，合併多個搜尋結果。",
-  "core.dataset.search.search mode": "搜索方式",
+  "core.dataset.search.search mode": "搜尋模式",
   "core.dataset.status.active": "已就緒",
   "core.dataset.status.syncing": "同步中",
   "core.dataset.test.Batch test": "批次測試",
@@ -1024,7 +1024,6 @@
   "question_feedback": "工單諮詢",
   "read_quote": "查看引用",
   "required": "必填",
-  "rerank_weight": "重排權重",
   "resume_failed": "恢復失敗",
   "select_reference_variable": "選擇引用變數",
   "share_link": "分享連結",

packages/web/package.json

@@ -4,8 +4,8 @@
   "dependencies": {
     "@chakra-ui/anatomy": "2.2.1",
     "@chakra-ui/icons": "2.1.1",
-    "@chakra-ui/next-js": "2.4.2",
+    "@chakra-ui/next-js": "2.1.5",
-    "@chakra-ui/react": "2.10.7",
+    "@chakra-ui/react": "2.8.1",
     "@chakra-ui/styled-system": "2.9.1",
     "@chakra-ui/system": "2.6.1",
     "@emotion/react": "11.11.1",
@@ -21,11 +21,11 @@
     "ahooks": "^3.7.11",
     "date-fns": "2.30.0",
     "dayjs": "^1.11.7",
-    "i18next": "23.16.8",
+    "i18next": "23.11.5",
     "js-cookie": "^3.0.5",
     "lexical": "0.12.6",
     "lodash": "^4.17.21",
-    "next-i18next": "15.4.2",
+    "next-i18next": "15.3.0",
     "papaparse": "^5.4.1",
     "react": "18.3.1",
     "react-beautiful-dnd": "^13.1.1",

projects/app/.env.template

@@ -52,8 +52,6 @@ USE_IP_LIMIT=false
 WORKFLOW_MAX_RUN_TIMES=500
 # 循环最大运行次数，避免极端的死循环情况
 WORKFLOW_MAX_LOOP_TIMES=50
-# 启用内网 IP 检查
-CHECK_INTERNAL_IP=false
 
 # 对话日志推送服务
 # # 日志服务地址

projects/app/next-i18next.config.js

@@ -1,6 +1,8 @@
+//next-i18next.config.js
 /**
  * @type {import('next-i18next').UserConfig}
  */
+
 module.exports = {
   i18n: {
     defaultLocale: 'en',

projects/app/next.config.js

@@ -1,4 +1,4 @@
-const { i18n } = require('./next-i18next.config.js');
+const { i18n } = require('./next-i18next.config');
 const path = require('path');
 const fs = require('fs');
 
@@ -30,6 +30,10 @@ const nextConfig = {
           test: /\.svg$/i,
           issuer: /\.[jt]sx?$/,
           use: ['@svgr/webpack']
+        },
+        {
+          test: /\.node$/,
+          use: [{ loader: 'nextjs-node-loader' }]
         }
       ]),
       exprContextCritical: false,
@@ -41,7 +45,6 @@ const nextConfig = {
     }
 
     if (isServer) {
-      config.externals.push('@node-rs/jieba');
       if (nextRuntime === 'nodejs') {
         const oldEntry = config.entry;
         config = {
@@ -76,15 +79,14 @@ const nextConfig = {
 
     return config;
   },
-  // 需要转译的包
+  transpilePackages: ['@fastgpt/*', 'ahooks'],
-  transpilePackages: ['@fastgpt/global', '@fastgpt/web', 'ahooks'],
   experimental: {
     // 优化 Server Components 的构建和运行，避免不必要的客户端打包。
     serverComponentsExternalPackages: [
       'mongoose',
       'pg',
-      '@zilliz/milvus2-sdk-node',
+      '@node-rs/jieba',
-      "tiktoken",
+      '@zilliz/milvus2-sdk-node'
     ],
     outputFileTracingRoot: path.join(__dirname, '../../'),
     instrumentationHook: true

projects/app/package.json

@@ -11,8 +11,8 @@
   "dependencies": {
     "@chakra-ui/anatomy": "2.2.1",
     "@chakra-ui/icons": "2.1.1",
-    "@chakra-ui/next-js": "2.4.2",
+    "@chakra-ui/next-js": "2.1.5",
-    "@chakra-ui/react": "2.10.7",
+    "@chakra-ui/react": "2.8.1",
     "@chakra-ui/styled-system": "2.9.1",
     "@chakra-ui/system": "2.6.1",
     "@emotion/react": "11.11.1",
@@ -23,7 +23,9 @@
     "@fastgpt/templates": "workspace:*",
     "@fastgpt/web": "workspace:*",
     "@fortaine/fetch-event-source": "^3.0.6",
+    "@node-rs/jieba": "1.10.0",
     "@tanstack/react-query": "^4.24.10",
+    "@types/nprogress": "^0.2.0",
     "ahooks": "^3.7.11",
     "axios": "^1.8.2",
     "date-fns": "2.30.0",
@@ -33,7 +35,7 @@
     "formidable": "^2.1.1",
     "framer-motion": "9.1.7",
     "hyperdown": "^2.4.29",
-    "i18next": "23.16.8",
+    "i18next": "23.11.5",
     "immer": "^9.0.19",
     "js-yaml": "^4.1.0",
     "json5": "^2.2.3",
@@ -42,8 +44,9 @@
     "lodash": "^4.17.21",
     "mermaid": "^10.2.3",
     "nanoid": "^5.1.3",
-    "next": "14.2.24",
+    "next": "14.2.21",
-    "next-i18next": "15.4.2",
+    "next-i18next": "15.3.0",
+    "nextjs-node-loader": "^1.1.5",
     "nprogress": "^0.2.0",
     "qrcode": "^1.5.4",
     "react": "18.3.1",
@@ -64,7 +67,6 @@
     "request-ip": "^3.3.0",
     "sass": "^1.58.3",
     "use-context-selector": "^1.4.4",
-    "@node-rs/jieba": "2.0.1",
     "zustand": "^4.3.5"
   },
   "devDependencies": {
@@ -74,7 +76,6 @@
     "@types/jsonwebtoken": "^9.0.3",
     "@types/lodash": "^4.14.191",
     "@types/node": "^20.14.2",
-    "@types/nprogress": "^0.2.0",
     "@types/qrcode": "^1.5.5",
     "@types/react": "18.3.1",
     "@types/react-dom": "18.3.0",
@@ -82,6 +83,7 @@
     "@types/request-ip": "^0.0.37",
     "eslint": "8.56.0",
     "eslint-config-next": "14.2.24",
+    "nextjs-node-loader": "^1.1.5",
     "typescript": "^5.1.3",
     "vitest": "^3.0.2"
   }

projects/app/public/imgs/workflow/cfr.svg

@@ -0,0 +1 @@
+<?xml version="1.0" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg t="1703041511149" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="7024" xmlns:xlink="http://www.w3.org/1999/xlink" width="128" height="128"><path d="M512.512 128a384 384 0 1 1-384 384 384 384 0 0 1 384-384z m0 96a288 288 0 1 0 288 288 288 288 0 0 0-288-288z" fill="#3671FD" opacity=".2" p-id="7025"></path><path d="M512.512 64a448 448 0 1 1-302.56 117.6A32.032 32.032 0 1 1 253.312 228.8a384 384 0 1 0 298.016-98.88l-6.816-0.608v140.032a32 32 0 0 1-28.256 31.776l-3.744 0.224a32 32 0 0 1-31.776-28.288l-0.224-3.712V64z" fill="#3671FD" p-id="7026"></path><path d="M336.32 313.376l216.224 165.344a52.288 52.288 0 1 1-73.28 73.312l-165.376-216.224a16 16 0 0 1 22.4-22.4z" fill="#FE9C23" p-id="7027"></path></svg>

projects/app/src/components/Markdown/index.tsx

@@ -85,7 +85,7 @@ export default React.memo(Markdown);
 function Code(e: any) {
   const { className, codeBlock, children } = e;
   const match = /language-(\w+)/.exec(className || '');
-  const codeType = match?.[1]?.toLowerCase();
+  const codeType = match?.[1];
 
   const strChildren = String(children);
 
@@ -96,7 +96,7 @@ function Code(e: any) {
     if (codeType === CodeClassNameEnum.guide) {
       return <ChatGuide text={strChildren} />;
     }
-    if (codeType === CodeClassNameEnum.questionguide) {
+    if (codeType === CodeClassNameEnum.questionGuide) {
       return <QuestionGuide text={strChildren} />;
     }
     if (codeType === CodeClassNameEnum.echarts) {
@@ -105,7 +105,7 @@ function Code(e: any) {
     if (codeType === CodeClassNameEnum.iframe) {
       return <IframeCodeBlock code={strChildren} />;
     }
-    if (codeType === CodeClassNameEnum.html || codeType === CodeClassNameEnum.svg) {
+    if (codeType && codeType.toLowerCase() === CodeClassNameEnum.html) {
       return (
         <IframeHtmlCodeBlock className={className} codeBlock={codeBlock} match={match}>
           {children}

projects/app/src/components/Markdown/utils.ts

@@ -1,6 +1,6 @@
 export enum CodeClassNameEnum {
   guide = 'guide',
-  questionguide = 'questionguide',
+  questionGuide = 'questionGuide',
   mermaid = 'mermaid',
   echarts = 'echarts',
   quote = 'quote',
@@ -8,7 +8,6 @@ export enum CodeClassNameEnum {
   latex = 'latex',
   iframe = 'iframe',
   html = 'html',
-  svg = 'svg',
   video = 'video',
   audio = 'audio'
 }

projects/app/src/components/common/NextHead/index.tsx

@@ -1,6 +1,6 @@
 import { LOGO_ICON } from '@fastgpt/global/common/system/constants';
 import Head from 'next/head';
-import React, { useMemo } from 'react';
+import React, { useEffect, useMemo } from 'react';
 
 const NextHead = ({ title, icon, desc }: { title?: string; icon?: string; desc?: string }) => {
   const formatIcon = useMemo(() => {
@@ -11,6 +11,13 @@ const NextHead = ({ title, icon, desc }: { title?: string; icon?: string; desc?:
     return LOGO_ICON;
   }, [icon]);
 
+  useEffect(() => {
+    // Force update document title
+    if (title) {
+      document.title = title;
+    }
+  }, [title]);
+
   return (
     <Head>
       <title>{title}</title>

projects/app/src/components/core/app/DatasetParamsModal.tsx

@@ -2,15 +2,13 @@ import React, { useEffect, useMemo, useState } from 'react';
 import {
   Box,
   Button,
+  Checkbox,
+  Divider,
   Flex,
-  HStack,
   ModalBody,
   ModalFooter,
   Switch,
-  Slider,
+  useTheme
-  SliderTrack,
-  SliderFilledTrack,
-  SliderThumb
 } from '@chakra-ui/react';
 import { useForm } from 'react-hook-form';
 import MyModal from '@fastgpt/web/components/common/MyModal';
@@ -19,18 +17,30 @@ import { useTranslation } from 'next-i18next';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
+import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constants';
+import MyRadio from '@/components/common/MyRadio';
+import MyIcon from '@fastgpt/web/components/common/Icon';
 import LightRowTabs from '@fastgpt/web/components/common/Tabs/LightRowTabs';
 import { useUserStore } from '@/web/support/user/useUserStore';
+import { useToast } from '@fastgpt/web/hooks/useToast';
 import SelectAiModel from '@/components/Select/AIModelSelector';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
 import MyTextarea from '@/components/common/Textarea/MyTextarea';
 import { defaultDatasetMaxTokens } from '@fastgpt/global/core/app/constants';
 import InputSlider from '@fastgpt/web/components/common/MySlider/InputSlider';
-import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
-import { AppDatasetSearchParamsType } from '@fastgpt/global/core/app/type';
-import MyIcon from '@fastgpt/web/components/common/Icon';
 
+export type DatasetParamsProps = {
+  searchMode: `${DatasetSearchModeEnum}`;
+  limit?: number;
+  similarity?: number;
+  usingReRank?: boolean;
+  datasetSearchUsingExtensionQuery?: boolean;
+  datasetSearchExtensionModel?: string;
+  datasetSearchExtensionBg?: string;
+
+  maxTokens?: number; // limit max tokens
+};
 enum SearchSettingTabEnum {
   searchMode = 'searchMode',
   limit = 'limit',
@@ -41,22 +51,17 @@ const DatasetParamsModal = ({
   searchMode = DatasetSearchModeEnum.embedding,
   limit,
   similarity,
-  embeddingWeight,
   usingReRank,
-  rerankModel,
+  maxTokens = defaultDatasetMaxTokens,
-  rerankWeight,
   datasetSearchUsingExtensionQuery,
   datasetSearchExtensionModel,
   datasetSearchExtensionBg,
-  maxTokens = defaultDatasetMaxTokens,
   onClose,
   onSuccess
-}: AppDatasetSearchParamsType & {
+}: DatasetParamsProps & { onClose: () => void; onSuccess: (e: DatasetParamsProps) => void }) => {
-  maxTokens?: number; // limit max tokens
-  onClose: () => void;
-  onSuccess: (e: AppDatasetSearchParamsType) => void;
-}) => {
   const { t } = useTranslation();
+  const theme = useTheme();
+  const { toast } = useToast();
   const { teamPlanStatus } = useUserStore();
   const { reRankModelList, llmModelList, defaultModels } = useSystemStore();
   const [refresh, setRefresh] = useState(false);
@@ -67,41 +72,28 @@ const DatasetParamsModal = ({
       value: item.model,
       label: item.name
     })))();
-  const reRankModelSelectList = (() =>
-    reRankModelList.map((item) => ({
-      value: item.model,
-      label: item.name
-    })))();
 
-  const { register, setValue, getValues, handleSubmit, watch } =
+  const { register, setValue, getValues, handleSubmit, watch } = useForm<DatasetParamsProps>({
-    useForm<AppDatasetSearchParamsType>({
     defaultValues: {
-        searchMode,
-        embeddingWeight: embeddingWeight || 0.5,
-        usingReRank: !!usingReRank && teamPlanStatus?.standardConstants?.permissionReRank !== false,
-        rerankModel: rerankModel || defaultModels?.rerank?.model,
-        rerankWeight: rerankWeight || 0.5,
       limit,
       similarity,
+      searchMode,
+      usingReRank: !!usingReRank && teamPlanStatus?.standardConstants?.permissionReRank !== false,
       datasetSearchUsingExtensionQuery,
       datasetSearchExtensionModel: datasetSearchExtensionModel || defaultModels.llm?.model,
       datasetSearchExtensionBg
     }
   });
-
-  const searchModeWatch = watch('searchMode');
-  const embeddingWeightWatch = watch('embeddingWeight');
-  const fullTextWeightWatch = useMemo(() => {
-    const val = 1 - (embeddingWeightWatch || 0.5);
-    return Number(val.toFixed(2));
-  }, [embeddingWeightWatch]);
-
   const datasetSearchUsingCfrForm = watch('datasetSearchUsingExtensionQuery');
   const queryExtensionModel = watch('datasetSearchExtensionModel');
-
+  const cfbBgDesc = watch('datasetSearchExtensionBg');
   const usingReRankWatch = watch('usingReRank');
-  const reRankModelWatch = watch('rerankModel');
+  const searchModeWatch = watch('searchMode');
-  const rerankWeightWatch = watch('rerankWeight');
+
+  const searchModeList = useMemo(() => {
+    const list = Object.values(DatasetSearchModeMap);
+    return list;
+  }, []);
 
   const showSimilarity = useMemo(() => {
     if (similarity === undefined) return false;
@@ -142,160 +134,93 @@ const DatasetParamsModal = ({
       title={t('common:core.dataset.search.Dataset Search Params')}
       w={['90vw', '550px']}
     >
-      <ModalBody flex={'auto'} overflow={'auto'} px={[4, 10]}>
+      <ModalBody flex={'auto'} overflow={'auto'}>
         <LightRowTabs<SearchSettingTabEnum>
           width={'100%'}
           mb={3}
           list={[
             {
-              icon: 'common/setting',
+              icon: 'modal/setting',
               label: t('common:core.dataset.search.search mode'),
               value: SearchSettingTabEnum.searchMode
             },
             {
-              icon: 'core/dataset/searchfilter',
+              icon: 'support/outlink/apikeyFill',
               label: t('common:core.dataset.search.Filter'),
               value: SearchSettingTabEnum.limit
             },
             {
               label: t('common:core.module.template.Query extension'),
               value: SearchSettingTabEnum.queryExtension,
-              icon: 'core/dataset/questionExtension'
+              icon: '/imgs/workflow/cfr.svg'
             }
           ]}
-          inlineStyles={{
-            borderBottomColor: 'myGray.200',
-            borderBottom: '1px solid'
-          }}
           value={currentTabType}
           onChange={setCurrentTabType}
         />
         {currentTabType === SearchSettingTabEnum.searchMode && (
-          <Box mt={3}>
+          <>
-            <LeftRadio<`${DatasetSearchModeEnum}`>
+            <MyRadio
-              py={2.5}
+              gridGap={2}
-              gridGap={4}
+              gridTemplateColumns={'repeat(1,1fr)'}
-              list={[
+              list={searchModeList}
-                {
+              value={getValues('searchMode')}
-                  title: t('common:core.dataset.search.mode.embedding'),
-                  desc: t('common:core.dataset.search.mode.embedding desc'),
-                  value: DatasetSearchModeEnum.embedding
-                },
-                {
-                  title: t('common:core.dataset.search.mode.fullTextRecall'),
-                  desc: t('common:core.dataset.search.mode.fullTextRecall desc'),
-                  value: DatasetSearchModeEnum.fullTextRecall
-                },
-                {
-                  title: t('common:core.dataset.search.mode.mixedRecall'),
-                  desc: t('common:core.dataset.search.mode.mixedRecall desc'),
-                  value: DatasetSearchModeEnum.mixedRecall,
-                  children: searchModeWatch === DatasetSearchModeEnum.mixedRecall && (
-                    <Box mt={3}>
-                      <HStack justifyContent={'space-between'}>
-                        <Flex alignItems={'center'}>
-                          <Box fontSize={'sm'} color={'myGray.900'}>
-                            {t('common:core.dataset.search.mode.embedding')}
-                          </Box>
-                          <Box fontSize={'xs'} color={'myGray.500'}>
-                            {embeddingWeightWatch}
-                          </Box>
-                        </Flex>
-                        <Flex alignItems={'center'}>
-                          <Box fontSize={'sm'} color={'myGray.900'}>
-                            {t('common:core.dataset.search.score.fullText')}
-                          </Box>
-                          <Box fontSize={'xs'} color={'myGray.500'}>
-                            {fullTextWeightWatch}
-                          </Box>
-                        </Flex>
-                      </HStack>
-                      <Slider
-                        defaultValue={embeddingWeightWatch}
-                        min={0.1}
-                        max={0.9}
-                        step={0.01}
               onChange={(e) => {
-                          setValue('embeddingWeight', Number(e.toFixed(2)));
+                setValue('searchMode', e as `${DatasetSearchModeEnum}`);
+                setRefresh(!refresh);
+              }}
+            />
+            <>
+              <Divider my={4} />
+              <Flex
+                alignItems={'center'}
+                cursor={'pointer'}
+                userSelect={'none'}
+                py={3}
+                px={4}
+                border={theme.borders.sm}
+                borderWidth={'1.5px'}
+                borderRadius={'md'}
+                position={'relative'}
+                {...(getValues('usingReRank')
+                  ? {
+                      borderColor: 'primary.400'
+                    }
+                  : {})}
+                onClick={(e) => {
+                  if (!showReRank) {
+                    return toast({
+                      status: 'warning',
+                      title: t('common:core.ai.Not deploy rerank model')
+                    });
+                  }
+                  if (
+                    teamPlanStatus?.standardConstants &&
+                    !teamPlanStatus?.standardConstants?.permissionReRank
+                  ) {
+                    return toast({
+                      status: 'warning',
+                      title: t('common:support.team.limit.No permission rerank')
+                    });
+                  }
+                  setValue('usingReRank', !getValues('usingReRank'));
+                  setRefresh((state) => !state);
                 }}
               >
-                        <SliderTrack bg={'#F9518E'}>
+                <MyIcon name="core/dataset/rerank" w={'18px'} mr={'14px'} />
-                          <SliderFilledTrack bg={'#3370FF'} />
+                <Box pr={2} color={'myGray.800'} flex={'1 0 0'}>
-                        </SliderTrack>
+                  <Box fontSize={'sm'}>{t('common:core.dataset.search.ReRank')}</Box>
-                        <SliderThumb boxShadow={'none'} bg={'none'}>
+                  <Box fontSize={'xs'} color={'myGray.500'}>
-                          <MyIcon transform={'translateY(10px)'} name={'sliderTag'} w={'1rem'} />
+                    {t('common:core.dataset.search.ReRank desc')}
-                        </SliderThumb>
-                      </Slider>
                   </Box>
-                  )
-                }
-              ]}
-              value={searchModeWatch}
-              onChange={(e) => {
-                setValue('searchMode', e);
-              }}
-            />
-            {/* Rerank */}
-            <>
-              <HStack mt={6} justifyContent={'space-between'}>
-                <FormLabel>
-                  {t('common:core.dataset.search.ReRank')}
-                  <QuestionTip ml={0.5} label={t('common:core.dataset.search.ReRank desc')} />
-                </FormLabel>
-                {!showReRank ? (
-                  <Box color={'myGray.500'} fontSize={'sm'}>
-                    {t('common:core.ai.Not deploy rerank model')}
                 </Box>
-                ) : teamPlanStatus?.standardConstants &&
+                <Box position={'relative'} w={'18px'} h={'18px'}>
-                  !teamPlanStatus?.standardConstants?.permissionReRank ? (
+                  <Checkbox colorScheme="primary" isChecked={getValues('usingReRank')} size="lg" />
-                  <Box color={'myGray.500'} fontSize={'sm'}>
+                  <Box position={'absolute'} top={0} right={0} bottom={0} left={0} zIndex={1}></Box>
-                    {t('common:support.team.limit.No permission rerank')}
                 </Box>
-                ) : (
+              </Flex>
-                  <Switch {...register('usingReRank')} />
-                )}
-              </HStack>
-              {usingReRankWatch && (
-                <>
-                  <HStack mt={3} justifyContent={'space-between'}>
-                    <Box fontSize={'sm'} flex={'0 0 100px'} color={'myGray.700'}>
-                      {t('common:rerank_weight')}
-                    </Box>
-                    <Box flex={'1 0 0'}>
-                      <InputSlider
-                        min={0.1}
-                        max={1}
-                        step={0.01}
-                        value={rerankWeightWatch}
-                        onChange={(val) => {
-                          setValue(
-                            NodeInputKeyEnum.datasetSearchRerankWeight,
-                            Number(val.toFixed(2))
-                          );
-                        }}
-                      />
-                    </Box>
-                  </HStack>
-                  <HStack mt={3}>
-                    <Box fontSize={'sm'} flex={'0 0 100px'} color={'myGray.700'}>
-                      {t('common:model.type.reRank')}
-                    </Box>
-                    <Box flex={'1 0 0'}>
-                      <SelectAiModel
-                        bg={'myGray.50'}
-                        h={'36px'}
-                        value={reRankModelWatch}
-                        list={reRankModelSelectList}
-                        onChange={(val) => {
-                          setValue(NodeInputKeyEnum.datasetSearchRerankModel, val);
-                        }}
-                      />
-                    </Box>
-                  </HStack>
             </>
-              )}
           </>
-          </Box>
         )}
         {currentTabType === SearchSettingTabEnum.limit && (
           <Box pt={5}>
@@ -337,7 +262,7 @@ const DatasetParamsModal = ({
                     }}
                   />
                 ) : (
-                  <Box color={'myGray.500'} fontSize={'sm'}>
+                  <Box color={'myGray.500'}>
                     {t('common:core.dataset.search.No support similarity')}
                   </Box>
                 )}

projects/app/src/components/core/app/InputGuideConfig.tsx

@@ -54,6 +54,7 @@ const InputGuideConfig = ({
   onChange: (e: ChatInputGuideConfigType) => void;
 }) => {
   const { t } = useTranslation();
+  const { chatT } = useI18n();
   const { isOpen, onOpen, onClose } = useDisclosure();
   const {
     isOpen: isOpenLexiconConfig,
@@ -86,11 +87,11 @@ const InputGuideConfig = ({
     <Flex alignItems={'center'}>
       <MyIcon name={'core/app/inputGuides'} mr={2} w={'20px'} />
       <Flex alignItems={'center'}>
-        <FormLabel color={'myGray.600'}>{t('chat:input_guide')}</FormLabel>
+        <FormLabel color={'myGray.600'}>{chatT('input_guide')}</FormLabel>
         <ChatFunctionTip type={'inputGuide'} />
       </Flex>
       <Box flex={1} />
-      <MyTooltip label={t('chat:config_input_guide')}>
+      <MyTooltip label={chatT('config_input_guide')}>
         <Button
           variant={'transparentBase'}
           iconSpacing={1}
@@ -103,7 +104,7 @@ const InputGuideConfig = ({
         </Button>
       </MyTooltip>
       <MyModal
-        title={t('chat:input_guide')}
+        title={chatT('input_guide')}
         iconSrc="core/app/inputGuides"
         isOpen={isOpen}
         onClose={onClose}
@@ -125,7 +126,7 @@ const InputGuideConfig = ({
           {isOpenQuestionGuide && (
             <>
               <Flex mt={8} alignItems={'center'}>
-                <FormLabel>{t('chat:input_guide_lexicon')}</FormLabel>
+                <FormLabel>{chatT('input_guide_lexicon')}</FormLabel>
                 <Box fontSize={'xs'} px={2} bg={'myGray.100'} ml={1} rounded={'full'}>
                   {total}
                 </Box>
@@ -143,7 +144,7 @@ const InputGuideConfig = ({
               </Flex>
               <>
                 <Flex mt={8} alignItems={'center'}>
-                  <FormLabel>{t('chat:custom_input_guide_url')}</FormLabel>
+                  <FormLabel>{chatT('custom_input_guide_url')}</FormLabel>
                   <Flex
                     onClick={() => window.open(getDocPath('/docs/guide/course/chat_input_guide/'))}
                     color={'primary.700'}
@@ -180,7 +181,7 @@ const InputGuideConfig = ({
 export default React.memo(InputGuideConfig);
 
 const LexiconConfigModal = ({ appId, onClose }: { appId: string; onClose: () => void }) => {
-  const { commonT } = useI18n();
+  const { chatT, commonT } = useI18n();
   const { t } = useTranslation();
   const { toast } = useToast();
   const { File, onOpen: onOpenSelectFile } = useSelectFile({
@@ -231,7 +232,7 @@ const LexiconConfigModal = ({ appId, onClose }: { appId: string; onClose: () =>
         if (res.insertLength < textList.length) {
           toast({
             status: 'warning',
-            title: t('chat:insert_input_guide,_some_data_already_exists', { len: res.insertLength })
+            title: chatT('insert_input_guide,_some_data_already_exists', { len: res.insertLength })
           });
         } else {
           toast({
@@ -300,7 +301,7 @@ const LexiconConfigModal = ({ appId, onClose }: { appId: string; onClose: () =>
 
   return (
     <MyModal
-      title={t('chat:config_input_guide_lexicon_title')}
+      title={chatT('config_input_guide_lexicon_title')}
       iconSrc="core/app/inputGuides"
       isOpen={true}
       onClose={onClose}
@@ -337,7 +338,7 @@ const LexiconConfigModal = ({ appId, onClose }: { appId: string; onClose: () =>
             });
           }}
         >
-          <QuestionTip ml={-2} label={t('chat:csv_input_lexicon_tip')} />
+          <QuestionTip ml={-2} label={chatT('csv_input_lexicon_tip')} />
         </Box>
       </Flex>
       <Box px={8}>
@@ -393,7 +394,7 @@ const LexiconConfigModal = ({ appId, onClose }: { appId: string; onClose: () =>
             <MyInput
               autoFocus
               rightIcon={<MyIcon name={'save'} w={'14px'} cursor={'pointer'} />}
-              placeholder={t('chat:new_input_guide_lexicon')}
+              placeholder={chatT('new_input_guide_lexicon')}
               onBlur={(e) => {
                 createNewData([e.target.value.trim()]);
               }}
@@ -410,7 +411,7 @@ const LexiconConfigModal = ({ appId, onClose }: { appId: string; onClose: () =>
         px={8}
         flex={'1 0 0'}
         fontSize={'sm'}
-        EmptyChildren={<EmptyTip text={t('chat:chat_input_guide_lexicon_is_empty')} />}
+        EmptyChildren={<EmptyTip text={chatT('chat_input_guide_lexicon_is_empty')} />}
       >
         {scrollDataList.map((data, index) => {
           const item = data.data;

projects/app/src/components/core/chat/ChatContainer/ChatBox/Input/InputGuideBox.tsx

@@ -22,6 +22,7 @@ export default function InputGuideBox({
   onSend: (text: string) => void;
 }) {
   const { t } = useTranslation();
+  const { chatT } = useI18n();
   const chatInputGuide = useContextSelector(ChatBoxContext, (v) => v.chatInputGuide);
   const outLinkAuthData = useContextSelector(ChatBoxContext, (v) => v.outLinkAuthData);
 
@@ -64,9 +65,9 @@ export default function InputGuideBox({
     >
       <Flex alignItems={'center'} fontSize={'sm'} color={'myGray.600'} gap={2} mb={2} px={2}>
         <MyIcon name={'union'} />
-        <Box>{t('chat:input_guide')}</Box>
+        <Box>{chatT('input_guide')}</Box>
       </Flex>
-      {data.map((item) => (
+      {data.map((item, index) => (
         <Flex
           alignItems={'center'}
           as={'li'}

projects/app/src/components/core/chat/ChatContainer/ChatBox/components/ChatItem.tsx

@@ -59,7 +59,7 @@ type Props = BasicProps & {
 const RenderQuestionGuide = ({ questionGuides }: { questionGuides: string[] }) => {
   return (
     <Markdown
-      source={`\`\`\`${CodeClassNameEnum.questionguide}
+      source={`\`\`\`${CodeClassNameEnum.questionGuide}
 ${JSON.stringify(questionGuides)}`}
     />
   );

projects/app/src/components/core/chat/components/AIResponseBox.tsx

@@ -299,7 +299,6 @@ const RenderUserFormInteractive = React.memo(function RenderFormInput({
             <MyNumberInput
               min={input.min}
               max={input.max}
-              defaultValue={input.defaultValue}
               isDisabled={interactive.params.submitted}
               bg={'white'}
               register={register}

projects/app/src/components/core/chat/components/WholeResponseModal.tsx

@@ -228,23 +228,10 @@ export const WholeResponseContent = ({
         {activeModule?.searchMode && (
           <Row
             label={t('common:core.dataset.search.search mode')}
-            rawDom={
+            // @ts-ignore
-              <Flex border={'base'} borderRadius={'md'} p={2}>
+            value={t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
-                <Box>
-                  {/* @ts-ignore */}
-                  {t(DatasetSearchModeMap[activeModule.searchMode]?.title)}
-                </Box>
-                {activeModule.embeddingWeight && (
-                  <>{`(${t('chat:response_hybrid_weight', {
-                    emb: activeModule.embeddingWeight,
-                    text: 1 - activeModule.embeddingWeight
-                  })})`}</>
-                )}
-              </Flex>
-            }
           />
         )}
-
         <Row
           label={t('common:core.chat.response.module similarity')}
           value={activeModule?.similarity}
@@ -252,19 +239,7 @@ export const WholeResponseContent = ({
         <Row label={t('common:core.chat.response.module limit')} value={activeModule?.limit} />
         <Row
           label={t('common:core.chat.response.search using reRank')}
-          rawDom={
+          value={`${activeModule?.searchUsingReRank}`}
-            <Box border={'base'} borderRadius={'md'} p={2}>
-              {activeModule?.searchUsingReRank ? (
-                activeModule?.rerankModel ? (
-                  <Box>{`${activeModule.rerankModel}: ${activeModule.rerankWeight}`}</Box>
-                ) : (
-                  'True'
-                )
-              ) : (
-                `False`
-              )}
-            </Box>
-          }
         />
         {activeModule.queryExtensionResult && (
           <>

projects/app/src/components/core/dataset/DatasetTypeTag.tsx

@@ -3,10 +3,11 @@ import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import React from 'react';
 import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constants';
-import { useTranslation } from 'next-i18next';
+import { useI18n } from '@/web/context/I18n';
 
 const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps) => {
-  const { t } = useTranslation();
+  const { datasetT } = useI18n();
+
   const item = DatasetTypeMap[type] || DatasetTypeMap['dataset'];
 
   return (
@@ -23,7 +24,8 @@ const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & Fle
       {...props}
     >
       <MyIcon name={item.icon as any} w={'16px'} mr={2} color={'myGray.400'} />
-      <Box>{t(item.label as any)}</Box>
+      {/* @ts-ignore */}
+      <Box>{datasetT(item.label)}</Box>
     </Flex>
   );
 };

projects/app/src/global/common/api/systemRes.d.ts

@@ -3,7 +3,7 @@ import type {
   EmbeddingModelItemType,
   AudioSpeechModels,
   STTModelType,
-  RerankModelItemType
+  ReRankModelItemType
 } from '@fastgpt/global/core/ai/model.d';
 
 import type { FastGPTFeConfigsType } from '@fastgpt/global/common/system/types/index.d';

projects/app/src/global/core/dataset/api.d.ts

@@ -63,13 +63,8 @@ export type SearchTestProps = {
   text: string;
   [NodeInputKeyEnum.datasetSimilarity]?: number;
   [NodeInputKeyEnum.datasetMaxTokens]?: number;
-
   [NodeInputKeyEnum.datasetSearchMode]?: `${DatasetSearchModeEnum}`;
-  [NodeInputKeyEnum.datasetSearchEmbeddingWeight]?: number;
-
   [NodeInputKeyEnum.datasetSearchUsingReRank]?: boolean;
-  [NodeInputKeyEnum.datasetSearchRerankModel]?: string;
-  [NodeInputKeyEnum.datasetSearchRerankWeight]?: number;
 
   [NodeInputKeyEnum.datasetSearchUsingExtensionQuery]?: boolean;
   [NodeInputKeyEnum.datasetSearchExtensionModel]?: string;

projects/app/src/pageComponents/account/TeamSelector.tsx

@@ -1,12 +1,14 @@
 import React, { useMemo } from 'react';
-import { Box, ButtonProps } from '@chakra-ui/react';
+import { Box, ButtonProps, Flex } from '@chakra-ui/react';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import { useTranslation } from 'next-i18next';
+import Avatar from '@fastgpt/web/components/common/Avatar';
 import { getTeamList, putSwitchTeam } from '@/web/support/user/team/api';
 import { TeamMemberStatusEnum } from '@fastgpt/global/support/user/team/constant';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import MySelect from '@fastgpt/web/components/common/MySelect';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
+import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useRouter } from 'next/router';
 
 const TeamSelector = ({
@@ -19,7 +21,7 @@ const TeamSelector = ({
 }) => {
   const { t } = useTranslation();
   const router = useRouter();
-  const { userInfo } = useUserStore();
+  const { userInfo, initUserInfo } = useUserStore();
   const { setLoading } = useSystemStore();
 
   const { data: myTeams = [] } = useRequest2(() => getTeamList(TeamMemberStatusEnum.active), {
@@ -31,11 +33,12 @@ const TeamSelector = ({
     async (teamId: string) => {
       setLoading(true);
       await putSwitchTeam(teamId);
+      return initUserInfo();
     },
     {
       onFinally: () => {
-        router.reload();
         setLoading(false);
+        onChange?.();
       },
       errorToast: t('common:user.team.Switch Team Failed')
     }

projects/app/src/pageComponents/account/bill/BillTable.tsx

@@ -220,7 +220,13 @@ function BillDetailModal({ bill, onClose }: { bill: BillSchemaType; onClose: ()
             {bill.metadata.payWay === 'balance' ? (
               t('user:bill.not_need_invoice')
             ) : (
-              <Box>{bill.hasInvoice ? t('account_bill:yes') : t('account_bill:no')}</Box>
+              <Box>
+                {
+                  (bill.metadata.payWay = bill.hasInvoice
+                    ? t('account_bill:yes')
+                    : t('account_bill:no'))
+                }
+              </Box>
             )}
           </Flex>
         )}

projects/app/src/pageComponents/account/model/ModelConfigTable.tsx

@@ -280,10 +280,6 @@ const ModelTable = ({ Tab }: { Tab: React.ReactNode }) => {
 
       isCustom: true,
       isActive: true,
-
-      isDefault: false,
-      isDefaultDatasetTextModel: false,
-      isDefaultDatasetImageModel: false,
       // @ts-ignore
       type
     });

projects/app/src/pageComponents/account/team/MemberTable.tsx

@@ -175,7 +175,22 @@ function MemberTable({ Tabs }: { Tabs: React.ReactNode }) {
               borderRadius={'md'}
               ml={3}
               leftIcon={<MyIcon name="common/inviteLight" w={'16px'} color={'white'} />}
-              onClick={onOpenInvite}
+              onClick={() => {
+                if (
+                  teamPlanStatus?.standardConstants?.maxTeamMember &&
+                  teamPlanStatus.standardConstants.maxTeamMember <= members.length
+                ) {
+                  toast({
+                    status: 'warning',
+                    title: t('common:user.team.Over Max Member Tip', {
+                      max: teamPlanStatus.standardConstants.maxTeamMember
+                    })
+                  });
+                  setNotSufficientModalType(TeamErrEnum.teamMemberOverSize);
+                } else {
+                  onOpenInvite();
+                }
+              }}
             >
               {t('account_team:user_team_invite_member')}
             </Button>

projects/app/src/pageComponents/app/detail/Publish/API/index.tsx

@@ -1,10 +1,12 @@
 import React, { useEffect, useState } from 'react';
 import ApiKeyTable from '@/components/support/apikey/Table';
 import { useTranslation } from 'next-i18next';
+import { Box } from '@chakra-ui/react';
+import { useI18n } from '@/web/context/I18n';
 
 const API = ({ appId }: { appId: string }) => {
-  const { t } = useTranslation();
+  const { publishT } = useI18n();
-  return <ApiKeyTable tips={t('publish:app_key_tips')} appId={appId} />;
+  return <ApiKeyTable tips={publishT('app_key_tips')} appId={appId} />;
 };
 
 export default API;

projects/app/src/pageComponents/app/detail/SimpleApp/EditForm.tsx

@@ -10,6 +10,7 @@ import {
   HStack
 } from '@chakra-ui/react';
 import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d';
+import type { DatasetSimpleItemType } from '@fastgpt/global/core/dataset/type.d';
 import { useRouter } from 'next/router';
 import { useTranslation } from 'next-i18next';
 
@@ -105,7 +106,6 @@ const EditForm = ({
   const tokenLimit = useMemo(() => {
     return selectedModel?.quoteMaxToken || 3000;
   }, [selectedModel?.quoteMaxToken]);
-
   // Force close image select when model not support vision
   useEffect(() => {
     if (!selectedModel.vision) {
@@ -434,6 +434,8 @@ const EditForm = ({
                 ...e
               }
             }));
+
+            console.dir(e);
           }}
         />
       )}

projects/app/src/pageComponents/app/detail/SimpleApp/index.tsx

@@ -28,6 +28,8 @@ const SimpleEdit = () => {
 
   // Init app form
   useMount(() => {
+    // show selected dataset
+
     if (appDetail.version !== 'v2') {
       return setAppForm(
         appWorkflow2Form({

projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/render/RenderInput/templates/SelectDatasetParams.tsx

@@ -6,14 +6,13 @@ import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 import { NodeInputKeyEnum } from '@fastgpt/global/core/workflow/constants';
 import MyIcon from '@fastgpt/web/components/common/Icon';
-import DatasetParamsModal from '@/components/core/app/DatasetParamsModal';
+import DatasetParamsModal, { DatasetParamsProps } from '@/components/core/app/DatasetParamsModal';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import SearchParamsTip from '@/components/core/dataset/SearchParamsTip';
 import { useContextSelector } from 'use-context-selector';
 import { WorkflowContext } from '@/pageComponents/app/detail/WorkflowComponents/context';
 import { getWebLLMModel } from '@/web/common/system/utils';
 import { defaultDatasetMaxTokens } from '@fastgpt/global/core/app/constants';
-import { AppDatasetSearchParamsType } from '@fastgpt/global/core/app/type';
 
 const SelectDatasetParam = ({ inputs = [], nodeId }: RenderInputProps) => {
   const onChangeNode = useContextSelector(WorkflowContext, (v) => v.onChangeNode);
@@ -22,14 +21,11 @@ const SelectDatasetParam = ({ inputs = [], nodeId }: RenderInputProps) => {
   const { t } = useTranslation();
   const { defaultModels } = useSystemStore();
 
-  const [data, setData] = useState<AppDatasetSearchParamsType>({
+  const [data, setData] = useState<DatasetParamsProps>({
     searchMode: DatasetSearchModeEnum.embedding,
-    embeddingWeight: 0.5,
+    limit: 5,
-    limit: 3000,
     similarity: 0.5,
     usingReRank: false,
-    rerankModel: defaultModels.llm?.model,
-    rerankWeight: 0.6,
     datasetSearchUsingExtensionQuery: true,
     datasetSearchExtensionModel: defaultModels.llm?.model,
     datasetSearchExtensionBg: ''

projects/app/src/pageComponents/app/list/TypeTag.tsx

@@ -3,32 +3,31 @@ import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
 import { useI18n } from '@/web/context/I18n';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import { Box, Flex } from '@chakra-ui/react';
-import { useTranslation } from 'next-i18next';
 
 const AppTypeTag = ({ type }: { type: AppTypeEnum }) => {
-  const { t } = useTranslation();
+  const { appT } = useI18n();
 
   const map = useRef({
     [AppTypeEnum.simple]: {
-      label: t('app:type.Simple bot'),
+      label: appT('type.Simple bot'),
       icon: 'core/app/type/simple',
       bg: '#DBF3FF',
       color: '#0884DD'
     },
     [AppTypeEnum.workflow]: {
-      label: t('app:type.Workflow bot'),
+      label: appT('type.Workflow bot'),
       icon: 'core/app/type/workflow',
       bg: '#E4E1FC',
       color: '#6F5DD7'
     },
     [AppTypeEnum.plugin]: {
-      label: t('app:type.Plugin'),
+      label: appT('type.Plugin'),
       icon: 'core/app/type/plugin',
       bg: '#D0F5EE',
       color: '#007E7C'
     },
     [AppTypeEnum.httpPlugin]: {
-      label: t('app:type.Http plugin'),
+      label: appT('type.Http plugin'),
       icon: 'core/app/type/httpPlugin',
       bg: '#FFE4EE',
       color: '#E82F72'

projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteItem.tsx

@@ -2,12 +2,13 @@ import Markdown from '@/components/Markdown';
 import { Box, Flex } from '@chakra-ui/react';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 import { Dispatch, MutableRefObject, SetStateAction, useState } from 'react';
-import { useTranslation } from 'next-i18next';
+import { useTranslation } from 'react-i18next';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useCopyData } from '@fastgpt/web/hooks/useCopyData';
 import InputDataModal from '@/pageComponents/dataset/detail/InputDataModal';
 
 const CollectionQuoteItem = ({
+  index,
   quoteRefs,
   quoteIndex,
   setQuoteIndex,
@@ -21,7 +22,8 @@ const CollectionQuoteItem = ({
   dataId,
   collectionId
 }: {
-  quoteRefs: MutableRefObject<Map<string, HTMLDivElement | null>>;
+  index: number;
+  quoteRefs: MutableRefObject<(HTMLDivElement | null)[]>;
   quoteIndex: number;
   setQuoteIndex: Dispatch<SetStateAction<number>>;
   refreshList: () => void;
@@ -43,7 +45,7 @@ const CollectionQuoteItem = ({
     <>
       <Box
         ref={(el: HTMLDivElement | null) => {
-          quoteRefs.current.set(dataId, el);
+          quoteRefs.current[index] = el;
         }}
         p={2}
         py={2}
@@ -161,7 +163,9 @@ const CollectionQuoteItem = ({
                 '0px 1px 2px 0px rgba(19, 51, 107, 0.05), 0px 0px 1px 0px rgba(19, 51, 107, 0.08)'
               }
               cursor={'pointer'}
-              onClick={() => copyData(`${q}${a ? '\n' + a : ''}`)}
+              onClick={() => {
+                copyData(q + '\n' + a);
+              }}
             >
               <MyIcon name="copy" w={'14px'} color={'myGray.500'} />
             </Flex>

projects/app/src/pageComponents/chat/ChatQuoteList/CollectionQuoteReader.tsx

@@ -3,7 +3,7 @@ import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useRouter } from 'next/router';
-import { useTranslation } from 'next-i18next';
+import { useTranslation } from 'react-i18next';
 import DownloadButton from './DownloadButton';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { downloadFetch } from '@/web/common/system/utils';
@@ -70,9 +70,8 @@ const CollectionReader = ({
       appId: metadata.appId,
       ...metadata.outLinkAuthData
     }),
-    [chatItemDataId, collectionId, metadata.appId, metadata.chatId, metadata.outLinkAuthData]
+    [metadata]
   );
-
   const {
     dataList: datasetDataList,
     isLoading,
@@ -109,22 +108,13 @@ const CollectionReader = ({
       url: '/api/core/dataset/collection/export',
       filename: 'data.csv',
       body: {
-        appId: metadata.appId,
+        collectionId: collectionId,
-        chatId: metadata.chatId,
+        chatItemDataId
-        chatItemDataId,
-        collectionId,
-        ...metadata.outLinkAuthData
       }
     });
   });
 
-  const handleRead = getCollectionSourceAndOpen({
+  const handleRead = getCollectionSourceAndOpen(metadata);
-    appId: metadata.appId,
-    chatId: metadata.chatId,
-    chatItemDataId,
-    collectionId,
-    ...metadata.outLinkAuthData
-  });
 
   return (
     <MyBox display={'flex'} flexDirection={'column'} h={'full'}>
@@ -266,10 +256,11 @@ const CollectionReader = ({
             {formatedDataList.map((item, index) => (
               <CollectionQuoteItem
                 key={item._id}
-                quoteRefs={itemRefs as React.MutableRefObject<Map<string, HTMLDivElement | null>>}
+                index={index}
+                quoteRefs={itemRefs as React.MutableRefObject<(HTMLDivElement | null)[]>}
                 quoteIndex={item.quoteIndex}
                 setQuoteIndex={setQuoteIndex}
-                refreshList={() => loadInitData({ scrollWhenFinish: false, refresh: true })}
+                refreshList={() => loadInitData(false)}
                 updated={item.updated}
                 isCurrentSelected={item.isCurrentSelected}
                 q={item.q}

projects/app/src/pageComponents/chat/ChatQuoteList/DownloadButton.tsx

@@ -1,5 +1,5 @@
 import MyMenu from '@fastgpt/web/components/common/MyMenu';
-import { useTranslation } from 'next-i18next';
+import { useTranslation } from 'react-i18next';
 import MyIconButton from '@fastgpt/web/components/common/Icon/button';
 
 const DownloadButton = ({

projects/app/src/pageComponents/chat/ChatQuoteList/QuoteItem.tsx

@@ -4,7 +4,7 @@ import MyIcon from '@fastgpt/web/components/common/Icon';
 import ScoreTag from './ScoreTag';
 import Markdown from '@/components/Markdown';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
-import { useTranslation } from 'next-i18next';
+import { useTranslation } from 'react-i18next';
 import { useCopyData } from '@fastgpt/web/hooks/useCopyData';
 
 const QuoteItem = ({

projects/app/src/pageComponents/chat/ChatQuoteList/QuoteReader.tsx

@@ -2,7 +2,7 @@ import { Box, Flex } from '@chakra-ui/react';
 import { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import MyBox from '@fastgpt/web/components/common/MyBox';
-import { useTranslation } from 'next-i18next';
+import { useTranslation } from 'react-i18next';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import QuoteItem from './QuoteItem';
 import { useMemo } from 'react';

projects/app/src/pageComponents/chat/ChatQuoteList/ScoreTag.tsx

@@ -2,7 +2,7 @@ import { ScoreItemType, scoreTheme } from '@/components/core/dataset/QuoteItem';
 import { Box, Flex, Progress } from '@chakra-ui/react';
 import { SearchScoreTypeMap } from '@fastgpt/global/core/dataset/constants';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
-import { useTranslation } from 'next-i18next';
+import { useTranslation } from 'react-i18next';
 
 const ScoreTag = (score: { primaryScore?: ScoreItemType; secondaryScore: ScoreItemType[] }) => {
   const { t } = useTranslation();

projects/app/src/pageComponents/dataset/ApiDatasetForm.tsx

@@ -60,7 +60,7 @@ const ApiDatasetForm = ({
             <Input
               bg={'myWhite.600'}
               placeholder={t('dataset:request_headers')}
-              maxLength={2000}
+              maxLength={200}
               {...register('apiServer.authorization')}
             />
           </Flex>

projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx

@@ -36,19 +36,19 @@ import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import { shadowLight } from '@fastgpt/web/styles/theme';
 import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import { useToast } from '@fastgpt/web/hooks/useToast';
 
 function DataProcess() {
   const { t } = useTranslation();
   const { feConfigs } = useSystemStore();
+  const { toast } = useToast();
 
   const { goToNext, processParamsForm, chunkSizeField, minChunkSize, maxChunkSize } =
     useContextSelector(DatasetImportContext, (v) => v);
   const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
-  const { setValue, register, watch } = processParamsForm;
+  const { getValues, setValue, register, watch } = processParamsForm;
-
   const trainingType = watch('trainingType');
   const chunkSettingMode = watch('chunkSettingMode');
-  const qaPrompt = watch('qaPrompt');
 
   const {
     isOpen: isOpenCustomPrompt,
@@ -65,7 +65,7 @@ function DataProcess() {
         value: key as DatasetCollectionDataProcessModeEnum,
         tooltip: t(value.tooltip as any)
       }));
-  }, [t]);
+  }, []);
 
   const Title = useCallback(({ title }: { title: string }) => {
     return (
@@ -284,7 +284,7 @@ function DataProcess() {
                                   }
                                 }}
                               >
-                                {qaPrompt}
+                                {getValues('qaPrompt')}
 
                                 <Box
                                   display={'none'}
@@ -333,6 +333,44 @@ function DataProcess() {
             </AccordionPanel>
           </AccordionItem>
 
+          {/* <AccordionItem mt={4} border={'none'}>
+            <Title title={t('dataset:import_model_config')} />
+            <AccordionPanel p={2} fontSize={'sm'}>
+              <Box>
+                <Box>{t('common:core.ai.model.Dataset Agent Model')}</Box>
+                <Box mt={1}>
+                  <AIModelSelector
+                    w={'100%'}
+                    value={llmModel}
+                    list={datasetModelList.map((item) => ({
+                      label: item.name,
+                      value: item.model
+                    }))}
+                    onChange={(e) => {
+                      setValue('llmModel', e);
+                    }}
+                  />
+                </Box>
+              </Box>
+              <Box pt={5}>
+                <Box>{t('dataset:vllm_model')}</Box>
+                <Box mt={1}>
+                  <AIModelSelector
+                    w={'100%'}
+                    value={vlmModel}
+                    list={vllmModelList.map((item) => ({
+                      label: item.name,
+                      value: item.model
+                    }))}
+                    onChange={(e) => {
+                      setValue('vlmModel', e);
+                    }}
+                  />
+                </Box>
+              </Box>
+            </AccordionPanel>
+          </AccordionItem> */}
+
           <Flex mt={5} gap={3} justifyContent={'flex-end'}>
             <Button
               onClick={() => {
@@ -347,7 +385,7 @@ function DataProcess() {
 
       {isOpenCustomPrompt && (
         <PromptTextarea
-          defaultValue={qaPrompt}
+          defaultValue={getValues('qaPrompt')}
           onChange={(e) => {
             setValue('qaPrompt', e);
           }}

projects/app/src/pageComponents/dataset/detail/InputDataModal.tsx

@@ -17,6 +17,7 @@ import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils';
 import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type';
 import DeleteIcon from '@fastgpt/web/components/common/Icon/delete';
 import { defaultCollectionDetail } from '@/web/core/dataset/constants';
+import { getDocPath } from '@/web/common/system/doc';
 import MyBox from '@fastgpt/web/components/common/MyBox';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
@@ -256,18 +257,16 @@ const InputDataModal = ({
           />
         </Box>
 
-        <Flex flex={'1 0 0'} h={['auto', '0']} gap={6} flexDir={['column', 'row']} px={[5, '0']}>
-          {/* Data */}
         <Flex
           pt={4}
-            pl={[0, '3.25rem']}
-            flexDir={'column'}
-            h={'100%'}
-            gap={3}
           flex={'1 0 0'}
-            w={['100%', 0]}
+          gap={6}
-            overflow={['unset', 'auto']}
+          flexDir={['column', 'row']}
+          overflow={'auto'}
+          px={[5, '3.25rem']}
         >
+          {/* Data */}
+          <Flex flexDir={'column'} h={'100%'} gap={3} flex={'1 0 0'}>
             <Flex flexDir={'column'} h={'100%'}>
               <FormLabel required mb={1} h={'30px'}>
                 {currentTab === TabEnum.chunk
@@ -316,13 +315,7 @@ const InputDataModal = ({
             )}
           </Flex>
           {/* Index */}
-          <Box
+          <Box flex={'1 0 0'}>
-            pt={4}
-            pr={[0, '3.25rem']}
-            flex={'1 0 0'}
-            w={['100%', 0]}
-            overflow={['unset', 'auto']}
-          >
             <Flex alignItems={'flex-start'} justifyContent={'space-between'} h={'30px'}>
               <FormLabel>
                 {t('common:dataset.data.edit.Index', {

projects/app/src/pageComponents/dataset/detail/NavBar.tsx

@@ -6,6 +6,7 @@ import { useRouter } from 'next/router';
 import { useContextSelector } from 'use-context-selector';
 import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
 import LightRowTabs from '@fastgpt/web/components/common/Tabs/LightRowTabs';
+import { useI18n } from '@/web/context/I18n';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import MyPopover from '@fastgpt/web/components/common/MyPopover';
 import ParentPaths from '@/components/common/ParentPaths';
@@ -21,6 +22,7 @@ export enum TabEnum {
 const NavBar = ({ currentTab }: { currentTab: TabEnum }) => {
   const theme = useTheme();
   const { t } = useTranslation();
+  const { datasetT } = useI18n();
   const router = useRouter();
   const query = router.query;
   const { isPc } = useSystem();
@@ -166,7 +168,7 @@ const NavBar = ({ currentTab }: { currentTab: TabEnum }) => {
                 {rebuildingCount > 0 && (
                   <Box mb={3}>
                     <Box fontSize={'sm'}>
-                      {t('dataset:rebuilding_index_count', { count: rebuildingCount })}
+                      {datasetT('rebuilding_index_count', { count: rebuildingCount })}
                     </Box>
                   </Box>
                 )}

projects/app/src/pageComponents/dataset/detail/Test.tsx

@@ -36,14 +36,9 @@ type FormType = {
   inputText: string;
   searchParams: {
     searchMode: `${DatasetSearchModeEnum}`;
-    embeddingWeight?: number;
-
-    usingReRank?: boolean;
-    rerankModel?: string;
-    rerankWeight?: number;
-
     similarity?: number;
     limit?: number;
+    usingReRank?: boolean;
     datasetSearchUsingExtensionQuery?: boolean;
     datasetSearchExtensionModel?: string;
     datasetSearchExtensionBg?: string;
@@ -58,6 +53,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
   const { pushDatasetTestItem } = useSearchTestStore();
   const [inputType, setInputType] = useState<'text' | 'file'>('text');
   const [datasetTestItem, setDatasetTestItem] = useState<SearchTestStoreItemType>();
+  const [refresh, setRefresh] = useState(false);
   const [isFocus, setIsFocus] = useState(false);
   const { File, onOpen } = useSelectFile({
     fileType: '.csv',
@@ -70,10 +66,7 @@ const Test = ({ datasetId }: { datasetId: string }) => {
       inputText: '',
       searchParams: {
         searchMode: DatasetSearchModeEnum.embedding,
-        embeddingWeight: 0.5,
         usingReRank: false,
-        rerankModel: defaultModels?.rerank?.model,
-        rerankWeight: 0.5,
         limit: 5000,
         similarity: 0,
         datasetSearchUsingExtensionQuery: false,
@@ -84,7 +77,6 @@ const Test = ({ datasetId }: { datasetId: string }) => {
   });
 
   const searchModeData = DatasetSearchModeMap[getValues(`searchParams.searchMode`)];
-  const searchParams = getValues('searchParams');
 
   const {
     isOpen: isOpenSelectMode,
@@ -302,14 +294,15 @@ const Test = ({ datasetId }: { datasetId: string }) => {
 
       {isOpenSelectMode && (
         <DatasetParamsModal
-          {...searchParams}
+          {...getValues('searchParams')}
           maxTokens={20000}
           onClose={onCloseSelectMode}
           onSuccess={(e) => {
             setValue('searchParams', {
-              ...searchParams,
+              ...getValues('searchParams'),
               ...e
             });
+            setRefresh((state) => !state);
           }}
         />
       )}

projects/app/src/pages/_app.tsx

@@ -15,7 +15,6 @@ import { ReactElement, useEffect } from 'react';
 import { NextPage } from 'next';
 import { getWebReqUrl } from '@fastgpt/web/common/system/utils';
 import SystemStoreContextProvider from '@fastgpt/web/context/useSystem';
-import { useRouter } from 'next/router';
 
 type NextPageWithLayout = NextPage & {
   setLayout?: (page: ReactElement) => JSX.Element;
@@ -24,15 +23,6 @@ type AppPropsWithLayout = AppProps & {
   Component: NextPageWithLayout;
 };
 
-// 哪些路由有自定义 Head
-const routesWithCustomHead = [
-  '/chat',
-  '/chat/share',
-  'chat/team',
-  '/app/detail/',
-  '/dataset/detail'
-];
-
 function App({ Component, pageProps }: AppPropsWithLayout) {
   const { feConfigs, scripts, title } = useInitApp();
   const { t } = useTranslation();
@@ -52,12 +42,8 @@ function App({ Component, pageProps }: AppPropsWithLayout) {
 
   const setLayout = Component.setLayout || ((page) => <>{page}</>);
 
-  const router = useRouter();
-  const showHead = !router?.pathname || !routesWithCustomHead.includes(router.pathname);
-
   return (
     <>
-      {showHead && (
       <NextHead
         title={title}
         desc={
@@ -67,8 +53,6 @@ function App({ Component, pageProps }: AppPropsWithLayout) {
         }
         icon={getWebReqUrl(feConfigs?.favicon || process.env.SYSTEM_FAVICON)}
       />
-      )}
-
       {scripts?.map((item, i) => <Script key={i} strategy="lazyOnload" {...item}></Script>)}
 
       <QueryClientContext>

projects/app/src/pages/_error.tsx

@@ -1,6 +1,6 @@
 import { useEffect } from 'react';
 import { useRouter } from 'next/router';
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { Box } from '@chakra-ui/react';
 import { TrackEventName } from '@/web/common/system/constants';

projects/app/src/pages/account/apikey.tsx

@@ -3,7 +3,7 @@ import ApiKeyTable from '@/components/support/apikey/Table';
 import { useTranslation } from 'next-i18next';
 import { Box } from '@chakra-ui/react';
 import AccountContainer, { TabEnum } from '@/pageComponents/account/AccountContainer';
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 
 const ApiKey = () => {
   const { t } = useTranslation();

projects/app/src/pages/account/bill/index.tsx

@@ -6,7 +6,7 @@ import { useTranslation } from 'next-i18next';
 import ApplyInvoiceModal from '@/pageComponents/account/bill/ApplyInvoiceModal';
 import { useRouter } from 'next/router';
 import AccountContainer from '@/pageComponents/account/AccountContainer';
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 
 export enum InvoiceTabEnum {
   bill = 'bill',

projects/app/src/pages/account/info/index.tsx

@@ -39,7 +39,7 @@ import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import { getWebReqUrl } from '@fastgpt/web/common/system/utils';
 import AccountContainer from '@/pageComponents/account/AccountContainer';
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 import { useRouter } from 'next/router';
 import TeamSelector from '@/pageComponents/account/TeamSelector';
 import { getWorkorderURL } from '@/web/common/workorder/api';
@@ -277,7 +277,7 @@ const MyInfo = ({ onOpenContact }: { onOpenContact: () => void }) => {
           <Flex mt={6} alignItems={'center'}>
             <Box {...labelStyles}>{t('account_info:user_team_team_name')}:&nbsp;</Box>
             <Flex flex={'1 0 0'} w={0} align={'center'}>
-              <TeamSelector height={'28px'} w={'100%'} showManage />
+              <TeamSelector height={'28px'} w={'100%'} showManage onChange={initUserInfo} />
             </Flex>
           </Flex>
         )}

projects/app/src/pages/account/inform.tsx

@@ -7,7 +7,7 @@ import { useLoading } from '@fastgpt/web/hooks/useLoading';
 import { useTranslation } from 'next-i18next';
 import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
 import AccountContainer from '@/pageComponents/account/AccountContainer';
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 
 const InformTable = () => {
   const { t } = useTranslation();

projects/app/src/pages/account/model/index.tsx

@@ -1,4 +1,4 @@
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 import React, { useMemo, useState } from 'react';
 import AccountContainer from '@/pageComponents/account/AccountContainer';
 import { Box, Flex } from '@chakra-ui/react';

projects/app/src/pages/account/promotion.tsx

@@ -26,7 +26,7 @@ import { useLoading } from '@fastgpt/web/hooks/useLoading';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
 import AccountContainer from '@/pageComponents/account/AccountContainer';
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 
 const Promotion = () => {
   const { t } = useTranslation();

projects/app/src/pages/account/setting.tsx

@@ -10,7 +10,7 @@ import { UserUpdateParams } from '@/types/user';
 import TimezoneSelect from '@fastgpt/web/components/common/MySelect/TimezoneSelect';
 import I18nLngSelector from '@/components/Select/I18nLngSelector';
 import AccountContainer from '@/pageComponents/account/AccountContainer';
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 
 const Individuation = () => {
   const { t } = useTranslation();

projects/app/src/pages/account/team/index.tsx

@@ -1,4 +1,4 @@
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 import AccountContainer from '@/pageComponents/account/AccountContainer';
 import { Box, Flex } from '@chakra-ui/react';
 import Icon from '@fastgpt/web/components/common/Icon';
@@ -48,7 +48,10 @@ const Team = () => {
   const { t } = useTranslation();
   const { userInfo } = useUserStore();
 
-  const { setEditTeamData, isLoading, teamSize } = useContextSelector(TeamContext, (v) => v);
+  const { setEditTeamData, isLoading, teamSize, refetchMembers } = useContextSelector(
+    TeamContext,
+    (v) => v
+  );
 
   const Tabs = useMemo(
     () => (

projects/app/src/pages/account/thirdParty/index.tsx

@@ -9,7 +9,7 @@ import dynamic from 'next/dynamic';
 import { useState, useMemo } from 'react';
 import WorkflowVariableModal from '@/pageComponents/account/thirdParty/WorkflowVariableModal';
 import { useToast } from '@fastgpt/web/hooks/useToast';
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { GET } from '@/web/common/api/request';
 import type { checkUsageResponse } from '@/pages/api/support/user/team/thirtdParty/checkUsage';

projects/app/src/pages/account/usage/index.tsx

@@ -9,7 +9,7 @@ import { useTranslation } from 'next-i18next';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import Avatar from '@fastgpt/web/components/common/Avatar';
 import AccountContainer from '@/pageComponents/account/AccountContainer';
-import { serviceSideProps } from '@/web/common/i18n/utils';
+import { serviceSideProps } from '@fastgpt/web/common/system/nextjs';
 import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
 import { getTeamMembers } from '@/web/support/user/team/api';
 import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs';

projects/app/src/pages/api/admin/initv4818.ts

@@ -1,7 +1,7 @@
 import { NextAPI } from '@/service/middleware/entry';
 import { delay } from '@fastgpt/global/common/system/utils';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
-import { jiebaSplit } from '@fastgpt/service/common/string/jieba/index';
+import { jiebaSplit } from '@fastgpt/service/common/string/jieba';
 import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
 import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
@@ -44,7 +44,7 @@ const restore = async () => {
     const data = await MongoDatasetData.findOne({ fullTextToken: { $exists: false } });
     if (!data) return;
 
-    data.fullTextToken = await jiebaSplit({ text: `${data.q}\n${data.a}`.trim() });
+    data.fullTextToken = jiebaSplit({ text: `${data.q}\n${data.a}`.trim() });
     await data.save();
 
     success++;

projects/app/src/pages/api/admin/initv491.ts

@@ -1,82 +0,0 @@
-import { NextAPI } from '@/service/middleware/entry';
-import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { NextApiRequest, NextApiResponse } from 'next';
-import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
-import { jiebaSplit } from '@fastgpt/service/common/string/jieba';
-import { addLog } from '@fastgpt/service/common/system/log';
-import { delay } from '@fastgpt/global/common/system/utils';
-import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
-import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
-import { DatasetDataTextSchemaType } from '@fastgpt/global/core/dataset/type';
-import type { AnyBulkWriteOperation } from '@fastgpt/service/common/mongo';
-
-const updateData = async () => {
-  let success = 0;
-
-  while (true) {
-    try {
-      const time = Date.now();
-      const data = await MongoDatasetData.find({
-        initJieba: { $exists: false },
-        updateTime: { $lte: time } // 只需要取旧的数据
-      })
-        .limit(1000)
-        .lean();
-      if (data.length === 0) {
-        console.log('更新分词完成');
-        break;
-      }
-
-      const dataTextOps: AnyBulkWriteOperation<DatasetDataTextSchemaType>[] = [];
-      const datasetDataIds: string[] = [];
-
-      // 先进行分词处理
-      for await (const item of data) {
-        const text = `${item.q} ${item.a}`.trim();
-        try {
-          const tokens = await jiebaSplit({ text });
-          dataTextOps.push({
-            updateOne: {
-              filter: { dataId: item._id },
-              update: { $set: { fullTextToken: tokens } }
-            }
-          });
-          datasetDataIds.push(item._id);
-        } catch (error) {
-          console.log(`分词处理错误: ${item._id}`, error);
-        }
-      }
-
-      await mongoSessionRun(async (session) => {
-        if (dataTextOps.length > 0) {
-          await MongoDatasetDataText.bulkWrite(dataTextOps, { session, ordered: true });
-        }
-        if (datasetDataIds.length > 0) {
-          await MongoDatasetData.updateMany(
-            { _id: { $in: datasetDataIds } },
-            { $set: { initJieba: true } },
-            {
-              session
-            }
-          );
-        }
-      });
-
-      success += dataTextOps.length;
-      console.log(`成功 ${success}`);
-    } catch (error) {
-      addLog.error('更新所有旧的 jieba 分词失败', error);
-      await delay(1000);
-    }
-  }
-};
-
-async function handler(req: NextApiRequest, _res: NextApiResponse) {
-  await authCert({ req, authRoot: true });
-
-  console.log('更新所有旧的 jieba 分词');
-  updateData();
-  return { success: true };
-}
-
-export default NextAPI(handler);