perf: api dataset code

packages/service/core/dataset/apiDataset/custom/api.ts

@@ -3,12 +3,11 @@ import type {
   ApiFileReadContentResponse,
   APIFileReadResponse,
   ApiDatasetDetailResponse,
-  APIFileServer,
-  APIFileItem
-} from '@fastgpt/global/core/dataset/apiDataset';
+  APIFileServer
+} from '@fastgpt/global/core/dataset/apiDataset/type';
 import axios, { type Method } from 'axios';
-import { addLog } from '../../../common/system/log';
-import { readFileRawTextByUrl } from '../read';
+import { addLog } from '../../../../common/system/log';
+import { readFileRawTextByUrl } from '../../read';
 import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type';
 import { type RequireOnlyOne } from '@fastgpt/global/common/type/utils';
 

packages/service/core/dataset/apiDataset/feishuDataset/api.ts

@@ -3,10 +3,10 @@ import type {
   ApiFileReadContentResponse,
   ApiDatasetDetailResponse,
   FeishuServer
-} from '@fastgpt/global/core/dataset/apiDataset';
+} from '@fastgpt/global/core/dataset/apiDataset/type';
 import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type';
 import axios, { type Method } from 'axios';
-import { addLog } from '../../../common/system/log';
+import { addLog } from '../../../../common/system/log';
 
 type ResponseDataType = {
   success: boolean;

packages/service/core/dataset/apiDataset/index.ts

@@ -1,18 +1,10 @@
-import type {
-  APIFileServer,
-  YuqueServer,
-  FeishuServer
-} from '@fastgpt/global/core/dataset/apiDataset';
-import { useApiDatasetRequest } from './api';
-import { useYuqueDatasetRequest } from '../yuqueDataset/api';
-import { useFeishuDatasetRequest } from '../feishuDataset/api';
+import { useApiDatasetRequest } from './custom/api';
+import { useYuqueDatasetRequest } from './yuqueDataset/api';
+import { useFeishuDatasetRequest } from './feishuDataset/api';
+import type { ApiDatasetServerType } from '@fastgpt/global/core/dataset/apiDataset/type';
 
-export const getApiDatasetRequest = async (data: {
-  apiServer?: APIFileServer;
-  yuqueServer?: YuqueServer;
-  feishuServer?: FeishuServer;
-}) => {
-  const { apiServer, yuqueServer, feishuServer } = data;
+export const getApiDatasetRequest = async (apiDatasetServer?: ApiDatasetServerType) => {
+  const { apiServer, yuqueServer, feishuServer } = apiDatasetServer || {};
 
   if (apiServer) {
     return useApiDatasetRequest({ apiServer });

packages/service/core/dataset/apiDataset/yuqueDataset/api.ts

@@ -3,9 +3,9 @@ import type {
   ApiFileReadContentResponse,
   YuqueServer,
   ApiDatasetDetailResponse
-} from '@fastgpt/global/core/dataset/apiDataset';
+} from '@fastgpt/global/core/dataset/apiDataset/type';
 import axios, { type Method } from 'axios';
-import { addLog } from '../../../common/system/log';
+import { addLog } from '../../../../common/system/log';
 import { type ParentIdType } from '@fastgpt/global/common/parentFolder/type';
 
 type ResponseDataType = {
@@ -105,7 +105,6 @@ export const useYuqueDatasetRequest = ({ yuqueServer }: { yuqueServer: YuqueServ
     if (!parentId) {
       if (yuqueServer.basePath) parentId = yuqueServer.basePath;
     }
-
     let files: APIFileItem[] = [];
 
     if (!parentId) {

packages/service/core/dataset/collection/utils.ts

@@ -157,9 +157,7 @@ export const syncCollection = async (collection: CollectionWithDatasetType) => {
     return {
       type: DatasetSourceReadTypeEnum.apiFile,
       sourceId,
-      apiServer: dataset.apiServer,
-      feishuServer: dataset.feishuServer,
-      yuqueServer: dataset.yuqueServer
+      apiDatasetServer: dataset.apiDatasetServer
     };
   })();
 

packages/service/core/dataset/read.ts

@@ -9,13 +9,9 @@ import { type TextSplitProps, splitText2Chunks } from '@fastgpt/global/common/st
 import axios from 'axios';
 import { readRawContentByFileBuffer } from '../../common/file/read/utils';
 import { parseFileExtensionFromUrl } from '@fastgpt/global/common/string/tools';
-import {
-  type APIFileServer,
-  type FeishuServer,
-  type YuqueServer
-} from '@fastgpt/global/core/dataset/apiDataset';
 import { getApiDatasetRequest } from './apiDataset';
 import Papa from 'papaparse';
+import type { ApiDatasetServerType } from '@fastgpt/global/core/dataset/apiDataset/type';
 
 export const readFileRawTextByUrl = async ({
   teamId,
@@ -69,9 +65,7 @@ export const readDatasetSourceRawText = async ({
   sourceId,
   selector,
   externalFileId,
-  apiServer,
-  feishuServer,
-  yuqueServer,
+  apiDatasetServer,
   customPdfParse,
   getFormatText
 }: {
@@ -84,9 +78,7 @@ export const readDatasetSourceRawText = async ({
 
   selector?: string; // link selector
   externalFileId?: string; // external file dataset
-  apiServer?: APIFileServer; // api dataset
-  feishuServer?: FeishuServer; // feishu dataset
-  yuqueServer?: YuqueServer; // yuque dataset
+  apiDatasetServer?: ApiDatasetServerType; // api dataset
 }): Promise<{
   title?: string;
   rawText: string;
@@ -128,9 +120,7 @@ export const readDatasetSourceRawText = async ({
     };
   } else if (type === DatasetSourceReadTypeEnum.apiFile) {
     const { title, rawText } = await readApiServerFileContent({
-      apiServer,
-      feishuServer,
-      yuqueServer,
+      apiDatasetServer,
       apiFileId: sourceId,
       teamId,
       tmbId
@@ -147,17 +137,13 @@ export const readDatasetSourceRawText = async ({
 };
 
 export const readApiServerFileContent = async ({
-  apiServer,
-  feishuServer,
-  yuqueServer,
+  apiDatasetServer,
   apiFileId,
   teamId,
   tmbId,
   customPdfParse
 }: {
-  apiServer?: APIFileServer;
-  feishuServer?: FeishuServer;
-  yuqueServer?: YuqueServer;
+  apiDatasetServer?: ApiDatasetServerType;
   apiFileId: string;
   teamId: string;
   tmbId: string;
@@ -166,13 +152,7 @@ export const readApiServerFileContent = async ({
   title?: string;
   rawText: string;
 }> => {
-  return (
-    await getApiDatasetRequest({
-      apiServer,
-      yuqueServer,
-      feishuServer
-    })
-  ).getFileContent({
+  return (await getApiDatasetRequest(apiDatasetServer)).getFileContent({
     teamId,
     tmbId,
     apiFileId,

packages/service/core/dataset/schema.ts

@@ -127,14 +127,16 @@ const DatasetSchema = new Schema({
     type: Boolean,
     default: true
   },
-  apiServer: Object,
-  feishuServer: Object,
-  yuqueServer: Object,
+
+  apiDatasetServer: Object,
 
   // abandoned
   autoSync: Boolean,
   externalReadUrl: String,
-  defaultPermission: Number
+  defaultPermission: Number,
+  apiServer: Object,
+  feishuServer: Object,
+  yuqueServer: Object
 });
 
 try {