perf: full text collection and search code;perf: rename function (#3519)

* perf: full text collection and search code

* perf: rename function

* perf: notify modal

* remove invalid code

* perf: sso login

* perf: pay process

packages/service/core/dataset/data/dataTextSchema.ts

@@ -0,0 +1,48 @@
+import { connectionMongo, getMongoModel } from '../../../common/mongo';
+const { Schema } = connectionMongo;
+import { DatasetDataSchemaType } from '@fastgpt/global/core/dataset/type.d';
+import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant';
+import { DatasetCollectionName } from '../schema';
+import { DatasetColCollectionName } from '../collection/schema';
+import { DatasetDataCollectionName } from './schema';
+
+export const DatasetDataTextCollectionName = 'dataset_data_texts';
+
+const DatasetDataTextSchema = new Schema({
+  teamId: {
+    type: Schema.Types.ObjectId,
+    ref: TeamCollectionName,
+    required: true
+  },
+  datasetId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetCollectionName,
+    required: true
+  },
+  collectionId: {
+    type: Schema.Types.ObjectId,
+    ref: DatasetColCollectionName,
+    required: true
+  },
+  dataId: {
+    type: String,
+    ref: DatasetDataCollectionName,
+    required: true
+  },
+  fullTextToken: {
+    type: String,
+    required: true
+  }
+});
+
+try {
+  DatasetDataTextSchema.index({ teamId: 1, datasetId: 1, fullTextToken: 'text' });
+  DatasetDataTextSchema.index({ dataId: 'hashed' });
+} catch (error) {
+  console.log(error);
+}
+
+export const MongoDatasetDataText = getMongoModel<DatasetDataSchemaType>(
+  DatasetDataTextCollectionName,
+  DatasetDataTextSchema
+);

packages/service/core/dataset/data/schema.ts

@@ -71,17 +71,8 @@ const DatasetDataSchema = new Schema({
     type: Number,
     default: 0
   },
-  inited: {
-    type: Boolean
-  },
-  rebuilding: Boolean
-});
-
-DatasetDataSchema.virtual('collection', {
-  ref: DatasetColCollectionName,
-  localField: 'collectionId',
-  foreignField: '_id',
-  justOne: true
+  rebuilding: Boolean,
+  inited: Boolean
 });
 
 try {
@@ -100,6 +91,7 @@ try {
   DatasetDataSchema.index({ updateTime: 1 });
   // rebuild data
   DatasetDataSchema.index({ rebuilding: 1, teamId: 1, datasetId: 1 });
+  DatasetDataSchema.index({ inited: 'hashed' });
 } catch (error) {
   console.log(error);
 }