V4.8.18 feature (#3565)

* feat: org CRUD (#3380)

* feat: add org schema

* feat: org manage UI

* feat: OrgInfoModal

* feat: org tree view

* feat: org management

* fix: init root org

* feat: org permission for app

* feat: org support for dataset

* fix: disable org role control

* styles: opt type signatures

* fix: remove unused permission

* feat: delete org collaborator

* perf: Team org ui (#3499)

* perf: org ui

* perf: org ui

* feat: org auth for app & dataset (#3498)

* feat: auth org resource permission

* feat: org auth support for app & dataset

* perf: org permission check (#3500)

* i18n (#3501)

* name

* i18n

* feat: support dataset changeOwner (#3483)

* feat: support dataset changeOwner

* chore: update dataset change owner api

* feat: permission manage UI for org (#3503)

* perf: password check;perf: image upload check;perf: sso login check (#3509)

* perf: password check

* perf: image upload check

* perf: sso login check

* force show update notification modal & fix login page text (#3512)

* fix login page English text

* update notification modal

* perf: notify account (#3515)

* perf(plugin): improve searXNG empty result handling and documentation (#3507)

* perf(plugin): improve searXNG empty result handling and documentation

* 修改了文档和代码部分无搜索的结果的反馈

* refactor: org pathId (#3516)

* optimize payment process (#3517)

* feat: support wecom sso (#3518)

* feat: support wecom sso

* chore: remove unused wecom js-sdk dependency

* fix qrcode script (#3520)

* fix qrcode script

* i18n

* perf: full text collection and search code;perf: rename function (#3519)

* perf: full text collection and search code

* perf: rename function

* perf: notify modal

* remove invalid code

* perf: sso login

* perf: pay process

* 4.8.18 test (#3524)

* perf: remove local token

* perf: index

* perf: file encoding;perf: leave team code;@c121914yu perf: full text search code (#3528)

* perf: text encoding

* perf: leave team code

* perf: full text search code

* fix: http status

* perf: embedding search and vector avatar

* perf: async read file (#3531)

* refactor: team permission  manager (#3535)

* perf: classify org, group and member

* refactor: team per manager

* fix: missing functions

* 4.8.18 test (#3543)

* perf: login check

* doc

* perf: llm model config

* perf: team clb config

* fix: MemberModal UI (#3553)

* fix: adapt MemberModal title and icon

* fix: adapt member modal

* fix: search input placeholder

* fix: add button text

* perf: org permission (#3556)

* docs:用户答疑的官方文档补充 (#3540)

* docs:用户答疑的官方文档补充

* 问题回答的内容修补

* share link random avatar (#3541)

* share link random avatar

* fix

* delete unused code

* share page avatar (#3558)

* feat: init 4818

* share page avatar

* feat: tmp upgrade code (#3559)

* feat: tmp upgrade code

* fulltext search test

* update action

* full text tmp code (#3561)

* full text tmp code

* fix: init

* fix: init

* remove tmp code

* remove tmp code

* 4818-alpha

* 4.8.18 test (#3562)

* full text tmp code

* fix: init

* upgrade code

* account log

* account log

* perf: dockerfile

* upgrade code

* chore: update docs app template submission (#3564)

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>
Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>

projects/app/src/service/core/dataset/data/controller.ts

@@ -12,6 +12,7 @@ import { DatasetDataItemType } from '@fastgpt/global/core/dataset/type';
 import { getVectorModel } from '@fastgpt/service/core/ai/model';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
 import { ClientSession } from '@fastgpt/service/common/mongo';
+import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
 
 /* insert data.
  * 1. create data id
@@ -42,7 +43,8 @@ export async function insertData2Dataset({
 
   const qaStr = getDefaultIndex({ q, a }).text;
 
-  // empty indexes check, if empty, create default index
+  // 1. Get vector indexes and insert
+  // Empty indexes check, if empty, create default index
   indexes =
     Array.isArray(indexes) && indexes.length > 0
       ? indexes.map((index) => ({
@@ -77,7 +79,7 @@ export async function insertData2Dataset({
     )
   );
 
-  // create mongo data
+  // 2. Create mongo data
   const [{ _id }] = await MongoDatasetData.create(
     [
       {
@@ -87,7 +89,8 @@ export async function insertData2Dataset({
         collectionId,
         q,
         a,
-        fullTextToken: jiebaSplit({ text: qaStr }),
+        // FullText tmp
+        // fullTextToken: jiebaSplit({ text: qaStr }),
         chunkIndex,
         indexes: indexes?.map((item, i) => ({
           ...item,
@@ -98,6 +101,20 @@ export async function insertData2Dataset({
     { session }
   );
 
+  // 3. Create mongo data text
+  await MongoDatasetDataText.create(
+    [
+      {
+        teamId,
+        datasetId,
+        collectionId,
+        dataId: _id,
+        fullTextToken: jiebaSplit({ text: qaStr })
+      }
+    ],
+    { session }
+  );
+
   return {
     insertId: _id,
     tokens: result.reduce((acc, cur) => acc + cur.tokens, 0)
@@ -225,11 +242,19 @@ export async function updateData2Dataset({
     // update mongo other data
     mongoData.q = q || mongoData.q;
     mongoData.a = a ?? mongoData.a;
-    mongoData.fullTextToken = jiebaSplit({ text: mongoData.q + mongoData.a });
+    // FullText tmp
+    // mongoData.fullTextToken = jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() });
     // @ts-ignore
     mongoData.indexes = newIndexes;
     await mongoData.save({ session });
 
+    // update mongo data text
+    await MongoDatasetDataText.updateOne(
+      { dataId: mongoData._id },
+      { fullTextToken: jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() }) },
+      { session }
+    );
+
     // delete vector
     const deleteIdList = patchResult
       .filter((item) => item.type === 'delete' || item.type === 'update')