4.8.19 test (#3584)

* faet: dataset search filter

* fix: scroll page

packages/global/core/workflow/constants.ts

@@ -152,6 +152,7 @@ export enum NodeInputKeyEnum {
   datasetSearchExtensionModel = 'datasetSearchExtensionModel',
   datasetSearchExtensionBg = 'datasetSearchExtensionBg',
   collectionFilterMatch = 'collectionFilterMatch',
+  authTmbId = 'authTmbId',
 
   // concat dataset
   datasetQuoteList = 'system_datasetQuoteList',

packages/global/core/workflow/runtime/type.d.ts

@@ -41,6 +41,10 @@ export type ChatDispatchProps = {
     teamId: string;
     tmbId: string; // App tmbId
   };
+  runningUserInfo: {
+    teamId: string;
+    tmbId: string;
+  };
   uid: string; // Who run this workflow
 
   chatId?: string;

packages/global/core/workflow/template/system/datasetSearch.ts

@@ -89,6 +89,13 @@ export const DatasetSearchModule: FlowNodeTemplateType = {
       valueType: WorkflowIOValueTypeEnum.string,
       value: ''
     },
+    {
+      key: NodeInputKeyEnum.authTmbId,
+      renderTypeList: [FlowNodeInputTypeEnum.hidden],
+      label: '',
+      valueType: WorkflowIOValueTypeEnum.boolean,
+      value: false
+    },
     {
       ...Input_Template_UserChatInput,
       toolDescription: i18nT('workflow:content_to_search')

packages/service/core/dataset/utils.ts

@@ -0,0 +1,39 @@
+import { getTmbInfoByTmbId } from '../../support/user/team/controller';
+import { getResourcePermission } from '../../support/permission/controller';
+import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
+import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
+
+// TODO: 需要优化成批量获取权限
+export const filterDatasetsByTmbId = async ({
+  datasetIds,
+  tmbId
+}: {
+  datasetIds: string[];
+  tmbId: string;
+}) => {
+  const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
+
+  // First get all permissions
+  const permissions = await Promise.all(
+    datasetIds.map(async (datasetId) => {
+      const per = await getResourcePermission({
+        teamId,
+        tmbId,
+        resourceId: datasetId,
+        resourceType: PerResourceTypeEnum.dataset
+      });
+
+      if (per === undefined) return false;
+
+      const datasetPer = new DatasetPermission({
+        per,
+        isOwner: tmbPer.isOwner
+      });
+
+      return datasetPer.hasReadPer;
+    })
+  );
+
+  // Then filter datasetIds based on permissions
+  return datasetIds.filter((_, index) => permissions[index]);
+};

packages/service/core/workflow/dispatch/dataset/search.ts

@@ -17,6 +17,7 @@ import { ChatNodeUsageType } from '@fastgpt/global/support/wallet/bill/type';
 import { checkTeamReRankPermission } from '../../../../support/permission/teamLimit';
 import { MongoDataset } from '../../../dataset/schema';
 import { i18nT } from '../../../../../web/i18n/utils';
+import { filterDatasetsByTmbId } from '../../../dataset/utils';
 
 type DatasetSearchProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.datasetSelectList]: SelectedDatasetType;
@@ -29,6 +30,7 @@ type DatasetSearchProps = ModuleDispatchProps<{
   [NodeInputKeyEnum.datasetSearchExtensionModel]: string;
   [NodeInputKeyEnum.datasetSearchExtensionBg]: string;
   [NodeInputKeyEnum.collectionFilterMatch]: string;
+  [NodeInputKeyEnum.authTmbId]: boolean;
 }>;
 export type DatasetSearchResponse = DispatchNodeResultType<{
   [NodeOutputKeyEnum.datasetQuoteQA]: SearchDataResponseItemType[];
@@ -39,6 +41,7 @@ export async function dispatchDatasetSearch(
 ): Promise<DatasetSearchResponse> {
   const {
     runningAppInfo: { teamId },
+    runningUserInfo: { tmbId },
     histories,
     node,
     params: {
@@ -52,7 +55,8 @@ export async function dispatchDatasetSearch(
       datasetSearchUsingExtensionQuery,
       datasetSearchExtensionModel,
       datasetSearchExtensionBg,
-      collectionFilterMatch
+      collectionFilterMatch,
+      authTmbId = false
     }
   } = props as DatasetSearchProps;
 
@@ -64,18 +68,20 @@ export async function dispatchDatasetSearch(
     return Promise.reject(i18nT('common:core.chat.error.Select dataset empty'));
   }
 
+  const emptyResult = {
+    quoteQA: [],
+    [DispatchNodeResponseKeyEnum.nodeResponse]: {
+      totalPoints: 0,
+      query: '',
+      limit,
+      searchMode
+    },
+    nodeDispatchUsages: [],
+    [DispatchNodeResponseKeyEnum.toolResponses]: []
+  };
+
   if (!userChatInput) {
-    return {
-      quoteQA: [],
-      [DispatchNodeResponseKeyEnum.nodeResponse]: {
-        totalPoints: 0,
-        query: '',
-        limit,
-        searchMode
-      },
-      nodeDispatchUsages: [],
-      [DispatchNodeResponseKeyEnum.toolResponses]: []
-    };
+    return emptyResult;
   }
 
   // query extension
@@ -83,13 +89,24 @@ export async function dispatchDatasetSearch(
     ? getLLMModel(datasetSearchExtensionModel)
     : undefined;
 
-  const { concatQueries, rewriteQuery, aiExtensionResult } = await datasetSearchQueryExtension({
-    query: userChatInput,
-    extensionModel,
-    extensionBg: datasetSearchExtensionBg,
-    histories: getHistories(6, histories)
-  });
+  const [{ concatQueries, rewriteQuery, aiExtensionResult }, datasetIds] = await Promise.all([
+    datasetSearchQueryExtension({
+      query: userChatInput,
+      extensionModel,
+      extensionBg: datasetSearchExtensionBg,
+      histories: getHistories(6, histories)
+    }),
+    authTmbId
+      ? filterDatasetsByTmbId({
+          datasetIds: datasets.map((item) => item.datasetId),
+          tmbId
+        })
+      : Promise.resolve(datasets.map((item) => item.datasetId))
+  ]);
 
+  if (datasetIds.length === 0) {
+    return emptyResult;
+  }
   // console.log(concatQueries, rewriteQuery, aiExtensionResult);
 
   // get vector
@@ -110,7 +127,7 @@ export async function dispatchDatasetSearch(
     model: vectorModel.model,
     similarity,
     limit,
-    datasetIds: datasets.map((item) => item.datasetId),
+    datasetIds,
     searchMode,
     usingReRank: usingReRank && (await checkTeamReRankPermission(teamId)),
     collectionFilterMatch

packages/service/support/user/utils.ts

@@ -79,19 +79,16 @@ export const checkWebSyncLimit = async ({
  */
 export async function addSourceMember<T extends { tmbId: string }>({
   list,
-  teamId,
   session
 }: {
   list: T[];
-  teamId?: string;
   session?: ClientSession;
 }): Promise<Array<T & { sourceMember: SourceMemberType }>> {
-  if (!list.length) return [];
+  if (!Array.isArray(list)) return [];
 
   const tmbList = await MongoTeamMember.find(
     {
-      _id: { $in: list.map((item) => String(item.tmbId)) },
-      ...(teamId && { teamId })
+      _id: { $in: list.map((item) => String(item.tmbId)) }
     },
     'tmbId name avatar status',
     {
@@ -103,9 +100,10 @@ export async function addSourceMember<T extends { tmbId: string }>({
     .map((item) => {
       const tmb = tmbList.find((tmb) => String(tmb._id) === String(item.tmbId));
       if (!tmb) return;
+
       return {
         ...item,
-        ...(tmb && { sourceMember: { name: tmb.name, avatar: tmb.avatar, status: tmb.status } })
+        sourceMember: { name: tmb.name, avatar: tmb.avatar, status: tmb.status }
       };
     })
     .filter(Boolean) as Array<T & { sourceMember: SourceMemberType }>;

packages/web/i18n/en/workflow.json

@@ -13,6 +13,8 @@
   "append_application_reply_to_history_as_new_context": "Append the application's reply to the history as new context",
   "application_call": "Application Call",
   "assigned_reply": "Assigned Reply",
+  "auth_tmb_id": "Auth member",
+  "auth_tmb_id_tip": "After it is turned on, when the application is released to the outside world, the knowledge base will be filtered based on whether the user has permission to the knowledge base.\n\nIf it is not enabled, the configured knowledge base will be searched directly without permission filtering.",
   "can_not_loop": "This node can't loop.",
   "choose_another_application_to_call": "Select another application to call",
   "classification_result": "Classification Result",

packages/web/i18n/zh-CN/workflow.json

@@ -13,6 +13,8 @@
   "append_application_reply_to_history_as_new_context": "将该应用回复内容拼接到历史记录中，作为新的上下文返回",
   "application_call": "应用调用",
   "assigned_reply": "指定回复",
+  "auth_tmb_id": "使用者鉴权",
+  "auth_tmb_id_tip": "开启后，对外发布该应用时，还会根据用户是否有该知识库权限进行知识库过滤。\n若未开启，则直接按配置的知识库进行检索，不进行权限过滤。",
   "can_not_loop": "该节点不支持循环嵌套",
   "choose_another_application_to_call": "选择一个其他应用进行调用",
   "classification_result": "分类结果",

packages/web/i18n/zh-Hant/workflow.json

@@ -13,6 +13,8 @@
   "append_application_reply_to_history_as_new_context": "將應用程式的回覆附加到歷史紀錄中，作為新的脈絡",
   "application_call": "應用程式呼叫",
   "assigned_reply": "指定回覆",
+  "auth_tmb_id": "使用者鑑權",
+  "auth_tmb_id_tip": "開啟後，對外發布應用程式時，也會根據使用者是否有該知識庫權限進行知識庫過濾。\n\n若未開啟，則直接按配置的知識庫進行檢索，不進行權限過濾。",
   "can_not_loop": "這個節點不能迴圈。",
   "choose_another_application_to_call": "選擇另一個應用程式來呼叫",
   "classification_result": "分類結果",