add env to check internal ip (#4187)

* fix: ts

* update jieba package

* add env to check internal ip

* package

* fix: jieba

* reset package

* update config

* fix: jieba package

* init shell

* init version

* change team reload

packages/service/core/dataset/data/dataTextSchema.ts

@@ -41,7 +41,7 @@ try {
     }
   );
   DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, collectionId: 1 });
-  DatasetDataTextSchema.index({ dataId: 1 }, { unique: true });
+  DatasetDataTextSchema.index({ dataId: 'hashed' });
 } catch (error) {
   console.log(error);
 }

packages/service/core/dataset/data/schema.ts

@@ -86,7 +86,8 @@ const DatasetDataSchema = new Schema({
 
   // Abandon
   fullTextToken: String,
-  initFullText: Boolean
+  initFullText: Boolean,
+  initJieba: Boolean
 });
 
 try {
@@ -103,6 +104,9 @@ try {
   DatasetDataSchema.index({ updateTime: 1 });
   // rebuild data
   DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
+
+  // 为查询 initJieba 字段不存在的数据添加索引
+  DatasetDataSchema.index({ initJieba: 1 }, { sparse: true });
 } catch (error) {
   console.log(error);
 }

packages/service/core/dataset/search/controller.ts

@@ -16,7 +16,7 @@ import { reRankRecall } from '../../../core/ai/rerank';
 import { countPromptTokens } from '../../../common/string/tiktoken/index';
 import { datasetSearchResultConcat } from '@fastgpt/global/core/dataset/search/utils';
 import { hashStr } from '@fastgpt/global/common/string/tools';
-import { jiebaSplit } from '../../../common/string/jieba';
+import { jiebaSplit } from '../../../common/string/jieba/index';
 import { getCollectionSourceData } from '@fastgpt/global/core/dataset/collection/utils';
 import { Types } from '../../../common/mongo';
 import json5 from 'json5';