4.8.19 test (#3584)

* faet: dataset search filter

* fix: scroll page

packages/service/core/dataset/utils.ts

@@ -0,0 +1,39 @@
+import { getTmbInfoByTmbId } from '../../support/user/team/controller';
+import { getResourcePermission } from '../../support/permission/controller';
+import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
+import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
+
+// TODO: 需要优化成批量获取权限
+export const filterDatasetsByTmbId = async ({
+  datasetIds,
+  tmbId
+}: {
+  datasetIds: string[];
+  tmbId: string;
+}) => {
+  const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
+
+  // First get all permissions
+  const permissions = await Promise.all(
+    datasetIds.map(async (datasetId) => {
+      const per = await getResourcePermission({
+        teamId,
+        tmbId,
+        resourceId: datasetId,
+        resourceType: PerResourceTypeEnum.dataset
+      });
+
+      if (per === undefined) return false;
+
+      const datasetPer = new DatasetPermission({
+        per,
+        isOwner: tmbPer.isOwner
+      });
+
+      return datasetPer.hasReadPer;
+    })
+  );
+
+  // Then filter datasetIds based on permissions
+  return datasetIds.filter((_, index) => permissions[index]);
+};

packages/service/core/workflow/dispatch/dataset/search.ts

@@ -17,6 +17,7 @@ import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
 import { checkTeamReRankPermission } from '../../../../support/permission/teamLimit';
 import { MongoDataset } from '../../../dataset/schema';
 import { i18nT } from '../../../../../web/i18n/utils';
+import { filterDatasetsByTmbId } from '../../../dataset/utils';
 
 type DatasetSearchProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
@@ -29,6 +30,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.datasetSearchExtensionModel]: string;
   [NodeInputKeyEnum.datasetSearchExtensionBg]: string;
   [NodeInputKeyEnum.collectionFilterMatch]: string;
+  [NodeInputKeyEnum.authTmbId]: boolean;
 }>;
 export type DatasetSearchResponse = DispatchNodeResultType<{
   [NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[];
@@ -39,6 +41,7 @@ export async function dispatchDatasetSearch(
 ): Promise<DatasetSearchResponse> {
   const {
     runningAppInfo: { teamId },
+    runningUserInfo: { tmbId },
     histories,
     node,
     params: {
@@ -52,7 +55,8 @@ export async function dispatchDatasetSearch(
       datasetSearchUsingExtensionQuery,
       datasetSearchExtensionModel,
       datasetSearchExtensionBg,
-      collectionFilterMatch
+      collectionFilterMatch,
+      authTmbId = false
     }
   } = props as DatasetSearchProps;
 
@@ -64,18 +68,20 @@ export async function dispatchDatasetSearch(
     return Promise.reject(i18nT('common:core.chat.error.Select dataset empty'));
   }
 
+  const emptyResult = {
+    quoteQA: [],
+    [DispatchNodeResponseKeyEnum.nodeResponse]: {
+      totalPoints: 0,
+      query: '',
+      limit,
+      searchMode
+    },
+    nodeDispatchUsages: [],
+    [DispatchNodeResponseKeyEnum.toolResponses]: []
+  };
+
   if (!userChatInput) {
-    return {
-      quoteQA: [],
-      [DispatchNodeResponseKeyEnum.nodeResponse]: {
-        totalPoints: 0,
-        query: '',
-        limit,
-        searchMode
-      },
-      nodeDispatchUsages: [],
-      [DispatchNodeResponseKeyEnum.toolResponses]: []
-    };
+    return emptyResult;
   }
 
   // query extension
@@ -83,13 +89,24 @@ export async function dispatchDatasetSearch(
     ? getLLMModel(datasetSearchExtensionModel)
     : undefined;
 
-  const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
-    query: userChatInput,
-    extensionModel,
-    extensionBg: datasetSearchExtensionBg,
-    histories: getHistories(6, histories)
-  });
+  const [{ concatQueries, rewriteQuery, aiExtensionResult }, datasetIds] = await Promise.all([
+    datasetSearchQueryExtension({
+      query: userChatInput,
+      extensionModel,
+      extensionBg: datasetSearchExtensionBg,
+      histories: getHistories(6, histories)
+    }),
+    authTmbId
+      ? filterDatasetsByTmbId({
+          datasetIds: datasets.map((item) => item.datasetId),
+          tmbId
+        })
+      : Promise.resolve(datasets.map((item) => item.datasetId))
+  ]);
 
+  if (datasetIds.length === 0) {
+    return emptyResult;
+  }
   // console.log(concatQueries, rewriteQuery, aiExtensionResult);
 
   // get vector
@@ -110,7 +127,7 @@ export async function dispatchDatasetSearch(
     model: vectorModel.model,
     similarity,
     limit,
-    datasetIds: datasets.map((item) => item.datasetId),
+    datasetIds,
     searchMode,
     usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
     collectionFilterMatch

packages/service/support/user/utils.ts

@@ -79,19 +79,16 @@ export const checkWebSyncLimit = async ({
  */
 export async function addSourceMember<T extends { tmbId: string }>({
   list,
-  teamId,
   session
 }: {
   list: T[];
-  teamId?: string;
   session?: ClientSession;
 }): Promise<Array<T & { sourceMember: SourceMemberType }>> {
-  if (!list.length) return [];
+  if (!Array.isArray(list)) return [];
 
   const tmbList = await MongoTeamMember.find(
     {
-      _id: { $in: list.map((item) => String(item.tmbId)) },
-      ...(teamId && { teamId })
+      _id: { $in: list.map((item) => String(item.tmbId)) }
     },
     'tmbId name avatar status',
     {
@@ -103,9 +100,10 @@ export async function addSourceMember<T extends { tmbId: string }>({
     .map((item) => {
       const tmb = tmbList.find((tmb) => String(tmb._id) === String(item.tmbId));
       if (!tmb) return;
+
       return {
         ...item,
-        ...(tmb && { sourceMember: { name: tmb.name, avatar: tmb.avatar, status: tmb.status } })
+        sourceMember: { name: tmb.name, avatar: tmb.avatar, status: tmb.status }
       };
     })
     .filter(Boolean) as Array<T & { sourceMember: SourceMemberType }>;