V4.8.16 dev (#3431)

* feat: add feishu & yuque dataset (#3379)

* feat: add feishu & yuque dataset

* fix ts

* fix ts

* move type position

* fix

* fix: merge interface

* fix

* feat: dingtalk sso support (#3408)

* fix: optional sso state

* feat: dingtalk bot

* feat: dingtalk sso login

* chore: move i18n to user namespace

* feat: dingtalk bot integration (#3415)

* feat: dingtalk bot integration

* docs: config dingtalk bot

* feat:sear XNG服务 (#3413)

* feat:sear XNG服务

* 补充了courseUrl

* 添加了官方文档

* 错误时返回情况修正了一下

* Tracks (#3420)

* feat: node intro

* feat: add domain track

* dingding sso login

* perf: api dataset code and add doc

* feat: tracks

* feat: searXNG plugins

* fix: ts

* feat: delete node tracks (#3423)

* fix: dingtalk bot GET verification (#3424)

* 4.8.16 test: fix: plugin inputs render；fix: ui offset (#3426)

* fix: ui offset

* perf: dingding talk

* fix: plugin inputs render

* feat: menu all folder (#3429)

* fix: recall code

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>

projects/app/src/pages/api/core/app/create.ts

@@ -16,6 +16,7 @@ import { authApp } from '@fastgpt/service/support/permission/app/auth';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
 import { MongoUser } from '@fastgpt/service/support/user/schema';
+import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils';
 
 export type CreateAppBody = {
   parentId?: ParentIdType;
@@ -35,7 +36,7 @@ async function handler(req: ApiRequestProps<CreateAppBody>) {
   }
 
   // 凭证校验
-  const [{ teamId, tmbId }] = await Promise.all([
+  const [{ teamId, tmbId, userId }] = await Promise.all([
     authUserPer({ req, authToken: true, per: WritePermissionVal }),
     ...(parentId
       ? [authApp({ req, appId: parentId, per: WritePermissionVal, authToken: true })]
@@ -62,6 +63,13 @@ async function handler(req: ApiRequestProps<CreateAppBody>) {
     username: user?.username
   });
 
+  pushTrack.createApp({
+    type,
+    uid: userId,
+    teamId,
+    tmbId
+  });
+
   return appId;
 }
 

projects/app/src/pages/api/core/app/del.ts

@@ -16,6 +16,8 @@ import { findAppAndAllChildren } from '@fastgpt/service/core/app/controller';
 import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
 import { ClientSession } from '@fastgpt/service/common/mongo';
 import { deleteChatFiles } from '@fastgpt/service/core/chat/controller';
+import { getAppLatestVersion } from '@fastgpt/service/core/app/version/controller';
+import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils';
 
 async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   const { appId } = req.query as { appId: string };
@@ -25,12 +27,20 @@ async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   }
 
   // Auth owner (folder owner, can delete all apps in the folder)
-  const { teamId } = await authApp({ req, authToken: true, appId, per: OwnerPermissionVal });
+  const { teamId, tmbId, userId, app } = await authApp({
+    req,
+    authToken: true,
+    appId,
+    per: OwnerPermissionVal
+  });
 
   await onDelOneApp({
     teamId,
     appId
   });
+
+  // Tracks
+  pushTrack.countAppNodes({ teamId, tmbId, uid: userId, appId });
 }
 
 export default NextAPI(handler);

projects/app/src/pages/api/core/app/httpPlugin/create.ts

@@ -8,6 +8,7 @@ import { NextAPI } from '@/service/middleware/entry';
 import { onCreateApp, type CreateAppBody } from '../create';
 import { AppSchema } from '@fastgpt/global/core/app/type';
 import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
+import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils';
 
 export type createHttpPluginQuery = {};
 
@@ -28,7 +29,11 @@ async function handler(
     return Promise.reject('缺少参数');
   }
 
-  const { teamId, tmbId } = await authUserPer({ req, authToken: true, per: WritePermissionVal });
+  const { teamId, tmbId, userId } = await authUserPer({
+    req,
+    authToken: true,
+    per: WritePermissionVal
+  });
 
   await mongoSessionRun(async (session) => {
     // create http plugin folder
@@ -62,6 +67,13 @@ async function handler(
     }
   });
 
+  pushTrack.createApp({
+    type: AppTypeEnum.httpPlugin,
+    uid: userId,
+    teamId,
+    tmbId
+  });
+
   return {};
 }
 

projects/app/src/pages/api/core/chat/init.ts

@@ -45,7 +45,9 @@ async function handler(
   const { nodes, chatConfig } = await getAppLatestVersion(app._id, app);
 
   const pluginInputs =
-    app?.modules?.find((node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput)?.inputs ?? [];
+    chat?.pluginInputs ??
+    nodes?.find((node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput)?.inputs ??
+    [];
 
   return {
     chatId,

projects/app/src/pages/api/core/chat/outLink/init.ts

@@ -35,6 +35,10 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
   }
 
   const { nodes, chatConfig } = await getAppLatestVersion(app._id, app);
+  const pluginInputs =
+    chat?.pluginInputs ??
+    nodes?.find((node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput)?.inputs ??
+    [];
 
   jsonRes<InitChatResponse>(res, {
     data: {
@@ -56,9 +60,7 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
         avatar: app.avatar,
         intro: app.intro,
         type: app.type,
-        pluginInputs:
-          app?.modules?.find((node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput)
-            ?.inputs ?? []
+        pluginInputs
       }
     }
   });

projects/app/src/pages/api/core/chat/team/init.ts

@@ -43,6 +43,10 @@ async function handler(req: ApiRequestProps<InitTeamChatProps>, res: NextApiResp
 
   // get app and history
   const { nodes, chatConfig } = await getAppLatestVersion(app._id, app);
+  const pluginInputs =
+    chat?.pluginInputs ??
+    nodes?.find((node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput)?.inputs ??
+    [];
 
   jsonRes<InitChatResponse>(res, {
     data: {
@@ -64,9 +68,7 @@ async function handler(req: ApiRequestProps<InitTeamChatProps>, res: NextApiResp
         avatar: app.avatar,
         intro: app.intro,
         type: app.type,
-        pluginInputs:
-          app?.modules?.find((node) => node.flowNodeType === FlowNodeTypeEnum.pluginInput)
-            ?.inputs ?? []
+        pluginInputs
       }
     }
   });

projects/app/src/pages/api/core/dataset/apiDataset/list.ts

@@ -1,4 +1,6 @@
+import { getFeishuAndYuqueDatasetFileList } from '@/service/core/dataset/apiDataset/controller';
 import { NextAPI } from '@/service/middleware/entry';
+import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import { ParentIdType } from '@fastgpt/global/common/parentFolder/type';
 import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
 import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
@@ -26,11 +28,21 @@ async function handler(req: NextApiRequest) {
   });
 
   const apiServer = dataset.apiServer;
-  if (!apiServer) {
-    return Promise.reject('apiServer is required');
+  const feishuServer = dataset.feishuServer;
+  const yuqueServer = dataset.yuqueServer;
+
+  if (apiServer) {
+    return useApiDatasetRequest({ apiServer }).listFiles({ searchKey, parentId });
+  }
+  if (feishuServer || yuqueServer) {
+    return getFeishuAndYuqueDatasetFileList({
+      feishuServer,
+      yuqueServer,
+      parentId
+    });
   }
 
-  return useApiDatasetRequest({ apiServer }).listFiles({ searchKey, parentId });
+  return Promise.reject(DatasetErrEnum.noApiServer);
 }
 
 export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/collection/create/apiCollection.ts

@@ -34,12 +34,8 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
   });
 
   const apiServer = dataset.apiServer;
-  if (!apiServer) {
-    return Promise.reject('Api server not found');
-  }
-  if (!apiFileId) {
-    return Promise.reject('ApiFileId not found');
-  }
+  const feishuServer = dataset.feishuServer;
+  const yuqueServer = dataset.yuqueServer;
 
   // Auth same apiFileId
   const storeCol = await MongoDatasetCollection.findOne(
@@ -57,6 +53,8 @@ async function handler(req: NextApiRequest): CreateCollectionResponse {
 
   const content = await readApiServerFileContent({
     apiServer,
+    feishuServer,
+    yuqueServer,
     apiFileId,
     teamId
   });

projects/app/src/pages/api/core/dataset/collection/create/reTrainingCollection.ts

@@ -66,7 +66,9 @@ async function handler(
       return {
         type: DatasetSourceReadTypeEnum.apiFile,
         sourceId: collection.apiFileId,
-        apiServer: collection.datasetId.apiServer
+        apiServer: collection.datasetId.apiServer,
+        feishuServer: collection.datasetId.feishuServer,
+        yuqueServer: collection.datasetId.yuqueServer
       };
     }
     if (collection.type === DatasetCollectionTypeEnum.externalFile) {

projects/app/src/pages/api/core/dataset/collection/read.ts

@@ -12,6 +12,7 @@ import { AIChatItemType, ChatHistoryItemResType } from '@fastgpt/global/core/cha
 import { authChatCrud } from '@/service/support/permission/auth/chat';
 import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller';
 import { useApiDatasetRequest } from '@fastgpt/service/core/dataset/apiDataset/api';
+import { POST } from '@fastgpt/service/common/api/plusRequest';
 
 export type readCollectionSourceQuery = {};
 
@@ -148,11 +149,25 @@ async function handler(
     }
     if (collection.type === DatasetCollectionTypeEnum.apiFile && collection.apiFileId) {
       const apiServer = collection.datasetId.apiServer;
-      if (!apiServer) return Promise.reject('apiServer not found');
+      const feishuServer = collection.datasetId.feishuServer;
+      const yuqueServer = collection.datasetId.yuqueServer;
 
-      return useApiDatasetRequest({ apiServer }).getFilePreviewUrl({
-        apiFileId: collection.apiFileId
-      });
+      if (apiServer) {
+        return useApiDatasetRequest({ apiServer }).getFilePreviewUrl({
+          apiFileId: collection.apiFileId
+        });
+      }
+
+      if (feishuServer || yuqueServer) {
+        return POST<string>(`/core/dataset/systemApiDataset`, {
+          type: 'read',
+          apiFileId: collection.apiFileId,
+          feishuServer,
+          yuqueServer
+        });
+      }
+
+      return '';
     }
     if (collection.type === DatasetCollectionTypeEnum.externalFile) {
       if (collection.externalFileId && collection.datasetId.externalReadUrl) {

projects/app/src/pages/api/core/dataset/create.ts

@@ -10,6 +10,7 @@ import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset';
 import type { ApiRequestProps } from '@fastgpt/service/type/next';
 import { parseParentIdInMongo } from '@fastgpt/global/common/parentFolder/utils';
 import { authDataset } from '@fastgpt/service/support/permission/dataset/auth';
+import { pushTrack } from '@fastgpt/service/common/middle/tracks/utils';
 
 export type DatasetCreateQuery = {};
 export type DatasetCreateBody = CreateDatasetParams;
@@ -26,11 +27,13 @@ async function handler(
     avatar,
     vectorModel = global.vectorModels[0].model,
     agentModel = getDatasetModel().model,
-    apiServer
+    apiServer,
+    feishuServer,
+    yuqueServer
   } = req.body;
 
   // auth
-  const [{ teamId, tmbId }] = await Promise.all([
+  const [{ teamId, tmbId, userId }] = await Promise.all([
     authUserPer({
       req,
       authToken: true,
@@ -70,7 +73,16 @@ async function handler(
     agentModel,
     avatar,
     type,
-    apiServer
+    apiServer,
+    feishuServer,
+    yuqueServer
+  });
+
+  pushTrack.createDataset({
+    type,
+    teamId,
+    tmbId,
+    uid: userId
   });
 
   return _id;

projects/app/src/pages/api/core/dataset/detail.ts

@@ -36,6 +36,19 @@ async function handler(req: ApiRequestProps<Query>): Promise<DatasetItemType> {
           authorization: ''
         }
       : undefined,
+    yuqueServer: dataset.yuqueServer
+      ? {
+          userId: dataset.yuqueServer.userId,
+          token: ''
+        }
+      : undefined,
+    feishuServer: dataset.feishuServer
+      ? {
+          appId: dataset.feishuServer.appId,
+          appSecret: '',
+          folderToken: dataset.feishuServer.folderToken
+        }
+      : undefined,
     permission,
     vectorModel: getVectorModel(dataset.vectorModel),
     agentModel: getLLMModel(dataset.agentModel)

projects/app/src/pages/api/core/dataset/file/getPreviewChunks.ts

@@ -50,7 +50,7 @@ async function handler(
     throw new Error('chunkSize is too large, should be less than 30000');
   }
 
-  const { teamId, apiServer } = await (async () => {
+  const { teamId, apiServer, feishuServer, yuqueServer } = await (async () => {
     if (type === DatasetSourceReadTypeEnum.fileLocal) {
       const res = await authCollectionFile({
         req,
@@ -72,7 +72,9 @@ async function handler(
     });
     return {
       teamId: dataset.teamId,
-      apiServer: dataset.apiServer
+      apiServer: dataset.apiServer,
+      feishuServer: dataset.feishuServer,
+      yuqueServer: dataset.yuqueServer
     };
   })();
 
@@ -83,6 +85,8 @@ async function handler(
     selector,
     isQAImport,
     apiServer,
+    feishuServer,
+    yuqueServer,
     externalFileId
   });
 

projects/app/src/pages/api/core/dataset/update.ts

@@ -57,6 +57,8 @@ async function handler(
     websiteConfig,
     externalReadUrl,
     apiServer,
+    yuqueServer,
+    feishuServer,
     status,
     autoSync
   } = req.body;
@@ -124,6 +126,13 @@ async function handler(
         ...(!!apiServer?.authorization && {
           'apiServer.authorization': apiServer.authorization
         }),
+        ...(!!yuqueServer?.userId && { 'yuqueServer.userId': yuqueServer.userId }),
+        ...(!!yuqueServer?.token && { 'yuqueServer.token': yuqueServer.token }),
+        ...(!!feishuServer?.appId && { 'feishuServer.appId': feishuServer.appId }),
+        ...(!!feishuServer?.appSecret && { 'feishuServer.appSecret': feishuServer.appSecret }),
+        ...(!!feishuServer?.folderToken && {
+          'feishuServer.folderToken': feishuServer.folderToken
+        }),
         ...(isMove && { inheritPermission: true }),
         ...(typeof autoSync === 'boolean' && { autoSync })
       },