perf: text splitter (#4584)

* perf: text splitter * update doc
2025-04-17 19:08:04 +08:00
parent cdbc1d5590
commit 22fc7dddfb
7 changed files with 103 additions and 473 deletions
--- a/projects/app/src/service/core/dataset/data/controller.ts
+++ b/projects/app/src/service/core/dataset/data/controller.ts
@@ -77,6 +77,7 @@ const formatIndexes = async ({

  // Recompute default indexes, Merge ids of the same index, reduce the number of rebuilds
  const defaultIndexes = getDefaultIndex({ q, a, indexSize });
+
  const concatDefaultIndexes = defaultIndexes.map((item) => {
    const oldIndex = indexes!.find((index) => index.text === item.text);
    if (oldIndex) {