perf: file encoding;perf: leave team code;@c121914yu perf: full text search code (#3528)

* perf: text encoding * perf: leave team code * perf: full text search code * fix: http status * perf: embedding search and vector avatar
2025-01-05 14:40:02 +08:00
parent 5465ca642f
commit 2bf1fce32a
24 changed files with 345 additions and 100 deletions
--- a/projects/app/src/pages/account/team/index.tsx
+++ b/projects/app/src/pages/account/team/index.tsx
@@ -76,14 +76,13 @@ const Team = () => {
    onClose: onCloseManageGroupMember
  } = useDisclosure();

-  const { runAsync: onLeaveTeam, loading: isLoadingLeaveTeam } = useRequest2(
-    async (teamId?: string) => {
-      if (!teamId) return;
+  const { runAsync: onLeaveTeam } = useRequest2(
+    async () => {
      const defaultTeam = myTeams.find((item) => item.defaultTeam) || myTeams[0];
      // change to personal team
      // get members
      onSwitchTeam(defaultTeam.teamId);
-      return delLeaveTeam(teamId);
+      return delLeaveTeam();
    },
    {
      onSuccess() {
@@ -242,10 +241,7 @@ const Team = () => {
                borderRadius={'md'}
                ml={3}
                leftIcon={<MyIcon name={'support/account/loginoutLight'} w={'14px'} />}
-                isLoading={isLoadingLeaveTeam}
-                onClick={() => {
-                  openLeaveConfirm(() => onLeaveTeam(userInfo?.team?.teamId))();
-                }}
+                onClick={() => openLeaveConfirm(onLeaveTeam)()}
              >
                {t('account_team:user_team_leave_team')}
              </Button>
--- a/projects/app/src/pages/api/admin/initv4818.ts
+++ b/projects/app/src/pages/api/admin/initv4818.ts
@@ -1,14 +1,104 @@
 import { NextAPI } from '@/service/middleware/entry';
+import { delay } from '@fastgpt/global/common/system/utils';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
+import { MongoDatasetDataText } from '@fastgpt/service/core/dataset/data/dataTextSchema';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { MongoUser } from '@fastgpt/service/support/user/schema';
-import { MongoTeam } from '@fastgpt/service/support/user/team/teamSchema';
 import { NextApiRequest, NextApiResponse } from 'next';

+/* 
+  简单版迁移：直接升级到最新镜像，会去除 MongoDatasetData 里的索引。直接执行这个脚本。
+  无缝迁移：
+    1. 先用 4.8.18-tmp 版本，会同时有 MongoDatasetData 和 MongoDatasetDataText 两个表和索引，依然是 MongoDatasetData 生效。会同步更新两张表数据。
+    2. 执行升级脚本，不要删除 MongoDatasetData 里的数据。
+    3. 切换正式版镜像，让 MongoDatasetDataText 生效。
+    4. 删除 MongoDatasetData 里的索引和多余字段。
+*/
+let success = 0;
 async function handler(req: NextApiRequest, res: NextApiResponse) {
  await authCert({ req, authRoot: true });

+  const batchSize = req.body.batchSize || 500;
+  success = 0;
+
+  const start = Date.now();
+  await initData(batchSize);
+  console.log('Init data time:', Date.now() - start);
+
+  success = 0;
+  await batchUpdateFields();
+
  return { success: true };
 }

 export default NextAPI(handler);
+
+const initData = async (batchSize: number) => {
+  try {
+    // 找到没有初始化的数据
+    const dataList = await MongoDatasetData.find(
+      {
+        initFullText: { $exists: false }
+      },
+      '_id teamId datasetId collectionId fullTextToken'
+    )
+      .limit(batchSize)
+      .lean();
+
+    if (dataList.length === 0) return;
+
+    await mongoSessionRun(async (session) => {
+      // 插入新数据
+      const result = await MongoDatasetDataText.insertMany(
+        dataList.map((item) => ({
+          teamId: item.teamId,
+          datasetId: item.datasetId,
+          collectionId: item.collectionId,
+          dataId: item._id,
+          fullTextToken: item.fullTextToken
+        })),
+        { ordered: false, session, lean: true }
+      );
+      // 把成功插入的新数据的 dataId 更新为已初始化
+      await MongoDatasetData.updateMany(
+        { _id: { $in: result.map((item) => item.dataId) } },
+        { $set: { initFullText: true }, $unset: { fullTextToken: 1 } },
+        { session }
+      );
+
+      success += result.length;
+
+      console.log('Success:', success);
+    });
+
+    await initData(batchSize);
+  } catch (error) {
+    console.log(error, '---');
+    await delay(500);
+    await initData(batchSize);
+  }
+};
+
+const batchUpdateFields = async (batchSize = 2000) => {
+  // Find documents that still have these fields
+  const documents = await MongoDatasetData.find({ initFullText: { $exists: true } }, '_id')
+    .limit(batchSize)
+    .lean();
+
+  if (documents.length === 0) return;
+
+  // Update in batches
+  await MongoDatasetData.updateMany(
+    { _id: { $in: documents.map((doc) => doc._id) } },
+    {
+      $unset: {
+        initFullText: 1
+        // fullTextToken: 1
+      }
+    }
+  );
+
+  success += documents.length;
+  console.log('Delete success:', success);
+  await batchUpdateFields(batchSize);
+};
--- a/projects/app/src/pages/api/common/file/upload.ts
+++ b/projects/app/src/pages/api/common/file/upload.ts
@@ -89,7 +89,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
      path: file.path,
      filename: file.originalname,
      contentType: file.mimetype,
-      encoding: file.encoding,
      metadata: metadata
    });

--- a/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts
+++ b/projects/app/src/pages/api/core/dataset/collection/create/localFile.ts
@@ -64,7 +64,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>): CreateCo
      path: file.path,
      filename: file.originalname,
      contentType: file.mimetype,
-      encoding: file.encoding,
      metadata: fileMetadata
    });

--- a/projects/app/src/pages/dataset/list/component/List.tsx
+++ b/projects/app/src/pages/dataset/list/component/List.tsx
@@ -31,6 +31,7 @@ import { useTranslation } from 'next-i18next';
 import { useUserStore } from '@/web/support/user/useUserStore';
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import SideTag from './SideTag';
+import { getModelProvider } from '@fastgpt/global/core/ai/provider';

 const EditResourceModal = dynamic(() => import('@/components/common/Modal/EditResourceModal'));

@@ -156,6 +157,8 @@ function List() {
        >
          {formatDatasets.map((dataset, index) => {
            const owner = members.find((v) => v.tmbId === dataset.tmbId);
+            const vectorModelAvatar = getModelProvider(dataset.vectorModel.provider)?.avatar;
+
            return (
              <MyTooltip
                key={dataset._id}
@@ -281,7 +284,7 @@ function List() {
                    <HStack>
                      {isPc && dataset.type !== DatasetTypeEnum.folder && (
                        <HStack spacing={1} className="time">
-                          <Avatar src={dataset.vectorModel.avatar} w={'0.85rem'} />
+                          <Avatar src={vectorModelAvatar} w={'0.85rem'} />
                          <Box color={'myGray.500'} fontSize={'mini'}>
                            {dataset.vectorModel.name}
                          </Box>
--- a/projects/app/src/service/core/dataset/data/controller.ts
+++ b/projects/app/src/service/core/dataset/data/controller.ts
@@ -89,7 +89,7 @@ export async function insertData2Dataset({
        collectionId,
        q,
        a,
-        fullTextToken: jiebaSplit({ text: qaStr }),
+        // fullTextToken: jiebaSplit({ text: qaStr }),
        chunkIndex,
        indexes: indexes?.map((item, i) => ({
          ...item,
@@ -241,7 +241,7 @@ export async function updateData2Dataset({
    // update mongo other data
    mongoData.q = q || mongoData.q;
    mongoData.a = a ?? mongoData.a;
-    mongoData.fullTextToken = jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() });
+    // mongoData.fullTextToken = jiebaSplit({ text: `${mongoData.q}\n${mongoData.a}`.trim() });
    // @ts-ignore
    mongoData.indexes = newIndexes;
    await mongoData.save({ session });
--- a/projects/app/src/web/support/user/team/api.ts
+++ b/projects/app/src/web/support/user/team/api.ts
@@ -37,8 +37,7 @@ export const delRemoveMember = (tmbId: string) =>
  DELETE(`/proApi/support/user/team/member/delete`, { tmbId });
 export const updateInviteResult = (data: UpdateInviteProps) =>
  PUT('/proApi/support/user/team/member/updateInvite', data);
-export const delLeaveTeam = (teamId: string) =>
-  DELETE('/proApi/support/user/team/member/leave', { teamId });
+export const delLeaveTeam = () => DELETE('/proApi/support/user/team/member/leave');

 export const getTeamClbs = () =>
  GET<ResourcePermissionType[]>(`/proApi/support/user/team/collaborator/list`);