perf: file encoding;perf: leave team code;@c121914yu perf: full text search code (#3528)

* perf: text encoding

* perf: leave team code

* perf: full text search code

* fix: http status

* perf: embedding search and vector avatar

packages/service/core/dataset/data/dataTextSchema.ts

@@ -25,7 +25,7 @@ const DatasetDataTextSchema = new Schema({
     required: true
   },
   dataId: {
-    type: String,
+    type: Schema.Types.ObjectId,
     ref: DatasetDataCollectionName,
     required: true
   },
@@ -37,7 +37,7 @@ const DatasetDataTextSchema = new Schema({
 
 try {
   DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
-  DatasetDataTextSchema.index({ dataId: 'hashed' });
+  DatasetDataTextSchema.index({ dataId: 1 }, { unique: true });
 } catch (error) {
   console.log(error);
 }

packages/service/core/dataset/data/schema.ts

@@ -39,10 +39,6 @@ const DatasetDataSchema = new Schema({
     type: String,
     default: ''
   },
-  fullTextToken: {
-    type: String,
-    default: ''
-  },
   indexes: {
     type: [
       {
@@ -72,7 +68,13 @@ const DatasetDataSchema = new Schema({
     default: 0
   },
   rebuilding: Boolean,
-  inited: Boolean
+
+  // Abandon
+  fullTextToken: {
+    type: String,
+    default: ''
+  },
+  initFullText: Boolean
 });
 
 try {
@@ -85,13 +87,14 @@ try {
     updateTime: -1
   });
   // full text index
-  DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
+  // DatasetDataSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
   // Recall vectors after data matching
   DatasetDataSchema.index({ teamId: 1, datasetId: 1, collectionId: 1, 'indexes.dataId': 1 });
   DatasetDataSchema.index({ updateTime: 1 });
   // rebuild data
   DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
-  DatasetDataSchema.index({ inited: 'hashed' });
+
+  DatasetDataSchema.index({ initFullText: 1 });
 } catch (error) {
   console.log(error);
 }