Dataset Permission (#1786)

* feat: dataset controllers

feat: dataset schema

fix: add missing type to dataset schema
Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: dataset list api

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: all dataset api

Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: new auth dataset method

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: use new auth method in detail, paths.
feat: add new param defaultPermission to create api

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: app auth params

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: use new auth method

Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: new auth collection and file method

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: dataset collection api new auth

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: create/*.ts auth

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: dataset auth

Signed-off-by: FinleyGe <m13203533462@163.com>

* fix: import paths

Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: dataset collaborator

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: dataset frontend

feat: dataset list frontend

feat: dataset detail
Signed-off-by: FinleyGe <m13203533462@163.com>

* feat: finish the dataset permission

fix: ts errors
Signed-off-by: FinleyGe <m13203533462@163.com>

* fix: empty response of collection api

Signed-off-by: FinleyGe <m13203533462@163.com>

* chore: adjust the code

* chore: adjust the code

* chore: i18n

* fix: ts error

* fix: fe CollectionCard permission

---------

Signed-off-by: FinleyGe <m13203533462@163.com>

packages/global/common/error/code/common.ts

@@ -4,7 +4,8 @@ import { ErrType } from '../errorCode';
 const startCode = 507000;
 export enum CommonErrEnum {
   fileNotFound = 'fileNotFound',
-  unAuthFile = 'unAuthFile'
+  unAuthFile = 'unAuthFile',
+  missingParams = 'missingParams'
 }
 const datasetErr = [
   {
@@ -14,6 +15,10 @@ const datasetErr = [
   {
     statusText: CommonErrEnum.unAuthFile,
     message: 'error.unAuthFile'
+  },
+  {
+    statusText: CommonErrEnum.missingParams,
+    message: 'error.missingParams'
   }
 ];
 export default datasetErr.reduce((acc, cur, index) => {

packages/global/common/error/code/dataset.ts

@@ -2,6 +2,7 @@ import { ErrType } from '../errorCode';
 
 /* dataset: 501000 */
 export enum DatasetErrEnum {
+  unExist = 'unExistDataset',
   unAuthDataset = 'unAuthDataset',
   unCreateCollection = 'unCreateCollection',
   unAuthDatasetCollection = 'unAuthDatasetCollection',
@@ -11,6 +12,10 @@ export enum DatasetErrEnum {
   unLinkCollection = 'unLinkCollection'
 }
 const datasetErr = [
+  {
+    statusText: DatasetErrEnum.unExist,
+    message: 'core.dataset.error.unExistDataset'
+  },
   {
     statusText: DatasetErrEnum.unAuthDataset,
     message: 'core.dataset.error.unAuthDataset'

packages/global/core/dataset/api.d.ts

@@ -9,12 +9,13 @@ export type DatasetUpdateBody = {
   name?: string;
   avatar?: string;
   intro?: string;
-  permission?: DatasetSchemaType['permission'];
+  permission?: DatasetSchemaType['permission']; // TODO: Should be deleted.
   agentModel?: LLMModelItemType;
   status?: DatasetSchemaType['status'];
 
   websiteConfig?: DatasetSchemaType['websiteConfig'];
   externalReadUrl?: DatasetSchemaType['externalReadUrl'];
+  defaultPermission?: DatasetSchemaType['defaultPermission'];
 };
 
 /* ================= collection ===================== */

packages/global/core/dataset/collaborator.d.ts

@@ -0,0 +1,11 @@
+import { UpdateClbPermissionProps } from '../../support/permission/collaborator';
+import { PermissionValueType } from '../../support/permission/type';
+
+export type UpdateDatasetCollaboratorBody = UpdateClbPermissionProps & {
+  datasetId: string;
+};
+
+export type DatasetCollaboratorDeleteParams = {
+  datasetId: string;
+  tmbId: string;
+};

packages/global/core/dataset/type.d.ts

@@ -1,3 +1,4 @@
+import { PermissionValueType } from 'support/permission/type';
 import type { LLMModelItemType, VectorModelItemType } from '../../core/ai/model.d';
 import { PermissionTypeEnum } from '../../support/permission/constant';
 import { PushDatasetDataChunkProps } from './api';
@@ -8,6 +9,8 @@ import {
   SearchScoreTypeEnum,
   TrainingModeEnum
 } from './constants';
+import { DatasetPermission } from '../../support/permission/dataset/controller';
+import { Permission } from '../../support/permission/controller';
 
 /* schema */
 export type DatasetSchemaType = {
@@ -24,7 +27,7 @@ export type DatasetSchemaType = {
   intro: string;
   type: DatasetTypeEnum;
   status: `${DatasetStatusEnum}`;
-  permission: `${PermissionTypeEnum}`;
+  permission: DatasetPermission;
 
   // metadata
   websiteConfig?: {
@@ -32,6 +35,7 @@ export type DatasetSchemaType = {
     selector: string;
   };
   externalReadUrl?: string;
+  defaultPermission: PermissionValueType;
 };
 
 export type DatasetCollectionSchemaType = {
@@ -132,24 +136,22 @@ export type DatasetListItemType = {
   name: string;
   intro: string;
   type: DatasetTypeEnum;
-  isOwner: boolean;
-  canWrite: boolean;
-  permission: `${PermissionTypeEnum}`;
+  permission: DatasetPermission;
   vectorModel: VectorModelItemType;
+  defaultPermission: PermissionValueType;
 };
+
 export type DatasetItemType = Omit<DatasetSchemaType, 'vectorModel' | 'agentModel'> & {
   vectorModel: VectorModelItemType;
   agentModel: LLMModelItemType;
-  isOwner: boolean;
-  canWrite: boolean;
 };
 
 /* ================= collection ===================== */
 export type DatasetCollectionItemType = CollectionWithDatasetType & {
-  canWrite: boolean;
   sourceName: string;
   sourceId?: string;
   file?: DatasetFileSchema;
+  permission: DatasetPermission;
 };
 
 /* ================= data ===================== */

packages/global/support/permission/dataset/constant.ts

@@ -0,0 +1,20 @@
+import { NullPermission, PermissionKeyEnum, PermissionList } from '../constant';
+
+export enum DatasetPermissionKeyEnum {}
+
+export const DatasetPermissionList = {
+  [PermissionKeyEnum.read]: {
+    ...PermissionList[PermissionKeyEnum.read],
+    description: '可查看知识库内容'
+  },
+  [PermissionKeyEnum.write]: {
+    ...PermissionList[PermissionKeyEnum.write],
+    description: '可增加和变更知识库内容'
+  },
+  [PermissionKeyEnum.manage]: {
+    ...PermissionList[PermissionKeyEnum.manage],
+    description: '可管理整个知识库数据和信息'
+  }
+};
+
+export const DatasetDefaultPermission = NullPermission;

packages/global/support/permission/dataset/controller.ts

@@ -0,0 +1,14 @@
+import { NullPermission } from '../constant';
+import { PerConstructPros, Permission } from '../controller';
+export class DatasetPermission extends Permission {
+  constructor(props?: PerConstructPros) {
+    if (!props) {
+      props = {
+        per: NullPermission
+      };
+    } else if (!props?.per) {
+      props.per = NullPermission;
+    }
+    super(props);
+  }
+}

packages/service/core/dataset/schema.ts

@@ -12,6 +12,7 @@ import {
   TeamMemberCollectionName
 } from '@fastgpt/global/support/user/team/constant';
 import { PermissionTypeEnum, PermissionTypeMap } from '@fastgpt/global/support/permission/constant';
+import { DatasetDefaultPermission } from '@fastgpt/global/support/permission/dataset/constant';
 
 export const DatasetCollectionName = 'datasets';
 
@@ -90,7 +91,11 @@ const DatasetSchema = new Schema({
       }
     }
   },
-  externalReadUrl: String
+  externalReadUrl: String,
+  defaultPermission: {
+    type: Number,
+    default: DatasetDefaultPermission
+  }
 });
 
 try {

packages/service/core/workflow/dispatch/plugin/run.ts

@@ -39,7 +39,6 @@ export const dispatchRunPlugin = async (props: RunPluginProps): Promise<RunPlugi
   if (source === PluginSourceEnum.personal) {
     await authAppByTmbId({
       appId: pluginId,
-      teamId: workflowApp.teamId,
       tmbId: workflowApp.tmbId,
       per: ReadPermissionVal
     });

packages/service/core/workflow/dispatch/tools/runApp.ts

@@ -2,7 +2,6 @@ import type { ChatItemType } from '@fastgpt/global/core/chat/type.d';
 import type { ModuleDispatchProps } from '@fastgpt/global/core/workflow/type/index.d';
 import { SelectAppItemType } from '@fastgpt/global/core/workflow/type/index.d';
 import { dispatchWorkFlow } from '../index';
-import { MongoApp } from '../../../../core/app/schema';
 import { responseWrite } from '../../../../common/response';
 import { ChatRoleEnum } from '@fastgpt/global/core/chat/constants';
 import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
@@ -48,7 +47,6 @@ export const dispatchAppRequest = async (props: Props): Promise<Response> => {
   // 检查该工作流的tmb是否有调用该app的权限（不是校验对话的人，是否有权限）
   const { app: appData } = await authAppByTmbId({
     appId: app.id,
-    teamId: workflowApp.teamId,
     tmbId: workflowApp.tmbId,
     per: ReadPermissionVal
   });

packages/service/support/permission/app/auth.ts

@@ -12,17 +12,15 @@ import { AuthResponseType } from '../type/auth.d';
 import { PermissionValueType } from '@fastgpt/global/support/permission/type';
 
 export const authAppByTmbId = async ({
-  teamId,
   tmbId,
   appId,
   per
 }: {
-  teamId: string;
   tmbId: string;
   appId: string;
   per: PermissionValueType;
 }) => {
-  const { permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
+  const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
 
   const app = await (async () => {
     // get app and per
@@ -68,10 +66,9 @@ export const authApp = async ({
   }
 > => {
   const result = await parseHeaderCert(props);
-  const { teamId, tmbId } = result;
+  const { tmbId } = result;
 
   const { app } = await authAppByTmbId({
-    teamId,
     tmbId,
     appId,
     per

packages/service/support/permission/auth/dataset.ts

@@ -1,201 +0,0 @@
-import { AuthModeType } from '../type';
-import { parseHeaderCert } from '../controller';
-import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
-import { MongoDataset } from '../../../core/dataset/schema';
-import { getCollectionWithDataset } from '../../../core/dataset/controller';
-import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant';
-import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant';
-import { AuthResponseType } from '@fastgpt/global/support/permission/type';
-import {
-  CollectionWithDatasetType,
-  DatasetFileSchema,
-  DatasetSchemaType
-} from '@fastgpt/global/core/dataset/type';
-import { getFileById } from '../../../common/file/gridfs/controller';
-import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
-import { getTmbInfoByTmbId } from '../../user/team/controller';
-import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
-import { MongoDatasetCollection } from '../../../core/dataset/collection/schema';
-
-export async function authDatasetByTmbId({
-  teamId,
-  tmbId,
-  datasetId,
-  per
-}: {
-  teamId: string;
-  tmbId: string;
-  datasetId: string;
-  per: AuthModeType['per'];
-}) {
-  const { role } = await getTmbInfoByTmbId({ tmbId });
-
-  const { dataset, isOwner, canWrite } = await (async () => {
-    const dataset = await MongoDataset.findOne({ _id: datasetId, teamId }).lean();
-
-    if (!dataset) {
-      return Promise.reject(DatasetErrEnum.unAuthDataset);
-    }
-
-    const isOwner =
-      role !== TeamMemberRoleEnum.visitor &&
-      (String(dataset.tmbId) === tmbId || role === TeamMemberRoleEnum.owner);
-    const canWrite =
-      isOwner ||
-      (role !== TeamMemberRoleEnum.visitor && dataset.permission === PermissionTypeEnum.public);
-    if (per === 'r') {
-      if (!isOwner && dataset.permission !== PermissionTypeEnum.public) {
-        return Promise.reject(DatasetErrEnum.unAuthDataset);
-      }
-    }
-    if (per === 'w' && !canWrite) {
-      return Promise.reject(DatasetErrEnum.unAuthDataset);
-    }
-    if (per === 'owner' && !isOwner) {
-      return Promise.reject(DatasetErrEnum.unAuthDataset);
-    }
-
-    return { dataset, isOwner, canWrite };
-  })();
-
-  return {
-    dataset,
-    isOwner,
-    canWrite
-  };
-}
-export async function authDataset({
-  datasetId,
-  per = 'owner',
-  ...props
-}: AuthModeType & {
-  datasetId: string;
-}): Promise<
-  AuthResponseType & {
-    dataset: DatasetSchemaType;
-  }
-> {
-  const result = await parseHeaderCert(props);
-  const { teamId, tmbId } = result;
-  const { dataset, isOwner, canWrite } = await authDatasetByTmbId({
-    teamId,
-    tmbId,
-    datasetId,
-    per
-  });
-
-  return {
-    ...result,
-    dataset,
-    isOwner,
-    canWrite
-  };
-}
-
-/* 
-   Read: in team and dataset permission is public
-   Write: in team, not visitor and dataset permission is public
-*/
-export async function authDatasetCollection({
-  collectionId,
-  per = 'owner',
-  ...props
-}: AuthModeType & {
-  collectionId: string;
-}): Promise<
-  AuthResponseType & {
-    collection: CollectionWithDatasetType;
-  }
-> {
-  const { teamId, tmbId } = await parseHeaderCert(props);
-  const { role } = await getTmbInfoByTmbId({ tmbId });
-
-  const { collection, isOwner, canWrite } = await (async () => {
-    const collection = await getCollectionWithDataset(collectionId);
-
-    if (!collection || String(collection.teamId) !== teamId) {
-      return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
-    }
-
-    const isOwner = String(collection.tmbId) === tmbId || role === TeamMemberRoleEnum.owner;
-    const canWrite =
-      isOwner ||
-      (role !== TeamMemberRoleEnum.visitor &&
-        collection.datasetId.permission === PermissionTypeEnum.public);
-
-    if (per === 'r') {
-      if (!isOwner && collection.datasetId.permission !== PermissionTypeEnum.public) {
-        return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
-      }
-    }
-    if (per === 'w' && !canWrite) {
-      return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
-    }
-    if (per === 'owner' && !isOwner) {
-      return Promise.reject(DatasetErrEnum.unAuthDatasetCollection);
-    }
-
-    return {
-      collection,
-      isOwner,
-      canWrite
-    };
-  })();
-
-  return {
-    teamId,
-    tmbId,
-    collection,
-    isOwner,
-    canWrite
-  };
-}
-
-export async function authDatasetFile({
-  fileId,
-  per = 'owner',
-  ...props
-}: AuthModeType & {
-  fileId: string;
-}): Promise<
-  AuthResponseType & {
-    file: DatasetFileSchema;
-  }
-> {
-  const { teamId, tmbId } = await parseHeaderCert(props);
-
-  const [file, collection] = await Promise.all([
-    getFileById({ bucketName: BucketNameEnum.dataset, fileId }),
-    MongoDatasetCollection.findOne({
-      teamId,
-      fileId
-    })
-  ]);
-
-  if (!file) {
-    return Promise.reject(CommonErrEnum.fileNotFound);
-  }
-
-  if (!collection) {
-    return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
-  }
-
-  // file role = collection role
-  try {
-    const { isOwner, canWrite } = await authDatasetCollection({
-      ...props,
-      collectionId: collection._id,
-      per
-    });
-
-    return {
-      teamId,
-      tmbId,
-      file,
-      isOwner,
-      canWrite
-    };
-  } catch (error) {
-    return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
-  }
-}

packages/service/support/permission/dataset/auth.ts

@@ -0,0 +1,213 @@
+import { PermissionValueType } from '@fastgpt/global/support/permission/type';
+import { getResourcePermission, parseHeaderCert } from '../controller';
+import { AuthPropsType, AuthResponseType } from '../type/auth';
+import {
+  CollectionWithDatasetType,
+  DatasetDataItemType,
+  DatasetFileSchema,
+  DatasetSchemaType
+} from '@fastgpt/global/core/dataset/type';
+import { getTmbInfoByTmbId } from '../../user/team/controller';
+import { MongoDataset } from '../../../core/dataset/schema';
+import { PerResourceTypeEnum } from '@fastgpt/global/support/permission/constant';
+import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
+import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
+import { getCollectionWithDataset } from '../../../core/dataset/controller';
+import { MongoDatasetCollection } from '../../../core/dataset/collection/schema';
+import { getFileById } from '../../../common/file/gridfs/controller';
+import { BucketNameEnum } from '@fastgpt/global/common/file/constants';
+import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { MongoDatasetData } from '../../../core/dataset/data/schema';
+
+export async function authDatasetByTmbId({
+  tmbId,
+  datasetId,
+  per
+}: {
+  tmbId: string;
+  datasetId: string;
+  per: PermissionValueType;
+}) {
+  const { teamId, permission: tmbPer } = await getTmbInfoByTmbId({ tmbId });
+
+  const dataset = await (async () => {
+    // get app and per
+    const [dataset, rp] = await Promise.all([
+      MongoDataset.findOne({ _id: datasetId, teamId }).lean(),
+      getResourcePermission({
+        teamId,
+        tmbId,
+        resourceId: datasetId,
+        resourceType: PerResourceTypeEnum.dataset
+      }) // this could be null
+    ]);
+
+    if (!dataset) {
+      return Promise.reject(DatasetErrEnum.unExist);
+    }
+
+    const isOwner = tmbPer.isOwner || String(dataset.tmbId) === tmbId;
+    const Per = new DatasetPermission({
+      per: rp?.permission ?? dataset.defaultPermission,
+      isOwner
+    });
+
+    if (!Per.checkPer(per)) {
+      return Promise.reject(DatasetErrEnum.unAuthDataset);
+    }
+
+    return {
+      ...dataset,
+      permission: Per
+    };
+  })();
+
+  return { dataset: dataset };
+}
+
+// Auth Dataset
+export async function authDataset({
+  datasetId,
+  per,
+  ...props
+}: AuthPropsType & {
+  datasetId: string;
+}): Promise<
+  AuthResponseType<DatasetPermission> & {
+    dataset: DatasetSchemaType;
+  }
+> {
+  const { teamId, tmbId } = await parseHeaderCert(props);
+
+  const { dataset } = await authDatasetByTmbId({
+    tmbId,
+    datasetId,
+    per
+  });
+
+  return {
+    teamId,
+    tmbId,
+    dataset,
+    permission: dataset.permission
+  };
+}
+
+// the temporary solution for authDatasetCollection is getting the
+export async function authDatasetCollection({
+  collectionId,
+  per,
+  ...props
+}: AuthPropsType & {
+  collectionId: string;
+}): Promise<
+  AuthResponseType<DatasetPermission> & {
+    collection: CollectionWithDatasetType;
+  }
+> {
+  const { teamId, tmbId } = await parseHeaderCert(props);
+  const collection = await getCollectionWithDataset(collectionId);
+
+  if (!collection) {
+    return Promise.reject(DatasetErrEnum.unExist);
+  }
+
+  const { dataset } = await authDatasetByTmbId({
+    tmbId,
+    datasetId: collection.datasetId._id,
+    per
+  });
+
+  return {
+    teamId,
+    tmbId,
+    collection,
+    permission: dataset.permission
+  };
+}
+
+export async function authDatasetFile({
+  fileId,
+  per,
+  ...props
+}: AuthPropsType & {
+  fileId: string;
+}): Promise<
+  AuthResponseType<DatasetPermission> & {
+    file: DatasetFileSchema;
+  }
+> {
+  const { teamId, tmbId } = await parseHeaderCert(props);
+
+  const [file, collection] = await Promise.all([
+    getFileById({ bucketName: BucketNameEnum.dataset, fileId }),
+    MongoDatasetCollection.findOne({
+      teamId,
+      fileId
+    })
+  ]);
+
+  if (!file) {
+    return Promise.reject(CommonErrEnum.fileNotFound);
+  }
+
+  if (!collection) {
+    return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
+  }
+
+  try {
+    const { permission } = await authDatasetCollection({
+      ...props,
+      collectionId: collection._id,
+      per
+    });
+
+    return {
+      teamId,
+      tmbId,
+      file,
+      permission
+    };
+  } catch (error) {
+    return Promise.reject(DatasetErrEnum.unAuthDatasetFile);
+  }
+}
+
+export async function authDatasetData({
+  dataId,
+  ...props
+}: AuthPropsType & {
+  dataId: string;
+}) {
+  // get mongo dataset.data
+  const datasetData = await MongoDatasetData.findById(dataId);
+
+  if (!datasetData) {
+    return Promise.reject('core.dataset.error.Data not found');
+  }
+
+  const result = await authDatasetCollection({
+    ...props,
+    collectionId: datasetData.collectionId
+  });
+
+  const data: DatasetDataItemType = {
+    id: String(datasetData._id),
+    teamId: datasetData.teamId,
+    q: datasetData.q,
+    a: datasetData.a,
+    chunkIndex: datasetData.chunkIndex,
+    indexes: datasetData.indexes,
+    datasetId: String(datasetData.datasetId),
+    collectionId: String(datasetData.collectionId),
+    sourceName: result.collection.name || '',
+    sourceId: result.collection?.fileId || result.collection?.rawLink,
+    isOwner: String(datasetData.tmbId) === result.tmbId,
+    canWrite: result.permission.hasWritePer
+  };
+
+  return {
+    ...result,
+    datasetData: data
+  };
+}

packages/service/support/permission/publish/authLink.ts

@@ -31,7 +31,6 @@ export async function authOutLinkCrud({
     }
 
     const { app } = await authAppByTmbId({
-      teamId,
       tmbId,
       appId: outLink.appId,
       per: ManagePermissionVal

packages/service/support/permission/type/auth.d.ts

@@ -11,11 +11,11 @@ export type AuthPropsType = {
   per: PermissionValueType;
 };
 
-export type AuthResponseType = {
+export type AuthResponseType<T = Permission> = {
   teamId: string;
   tmbId: string;
   authType?: `${AuthUserTypeEnum}`;
   appId?: string;
   apikey?: string;
-  permission: Permission;
+  permission: T;
 };