V4.8.15 feature (#3331)

* feat: add customize toolkit (#3205)

* chaoyang

* fix-auth

* add toolkit

* add order

* plugin usage

* fix

* delete console:

* Fix: Fix fullscreen preview top positioning and improve Markdown rendering logic (#3247)

* 完成任务：修复全屏预览顶部固定问题，优化 Markdown 渲染逻辑

* 有问题修改

* 问题再修改

* 修正问题

* fix: plugin standalone display issue (#3254)

* 4.8.15 test (#3246)

* o1 config

* perf: system plugin code

* 调整系统插件代码。增加html 渲染安全配置。 (#3258)

* perf: base64 picker

* perf: list app or dataset

* perf: plugin config code

* 小窗适配等问题 (#3257)

* 小窗适配等问题

* git问题

* 小窗剩余问题

* feat: system plugin auth and lock version (#3265)

* feat: system plugin auth and lock version

* update comment

* 4.8.15 test (#3267)

* tmp log

* perf: login direct

* perf: iframe html code

* remove log

* fix: plugin standalone display (#3277)

* refactor: 页面拆分&i18n拆分 (#3281)

* refactor: account组件拆成独立页面

* script: 新增i18n json文件创建脚本

* refactor: 页面i18n拆分

* i18n: add en&hant

* 4.8.15 test (#3285)

* tmp log

* remove log

* fix: watch avatar refresh

* perf: i18n code

* fix(plugin): use intro instead of userguide (#3290)

* Universal SSO (#3292)

* tmp log

* remove log

* feat: common oauth

* readme

* perf: sso provider

* remove sso code

* perf: refresh plugins

* feat: add api dataset (#3272)

* add api-dataset

* fix api-dataset

* fix api dataset

* fix ts

* perf: create collection code (#3301)

* tmp log

* remove log

* perf: i18n change

* update version doc

* feat: question guide from chatId

* perf: create collection code

* fix: request api

* fix: request api

* fix: tts auth and response type (#3303)

* perf: md splitter

* fix: tts auth and response type

* fix: api file dataset (#3307)

* perf: api dataset init (#3310)

* perf: collection schema

* perf: api dataset init

* refactor: 团队管理独立页面 (#3302)

* ui: 团队管理独立页面

* 代码优化

* fix

* perf: sync collection and ui check (#3314)

* perf: sync collection

* remove script

* perf: update api server

* perf: api dataset parent

* perf: team ui

* perf: team 18n

* update team ui

* perf: ui check

* perf: i18n

* fix: debug variables & cronjob & system plugin callback load (#3315)

* fix: debug variables & cronjob & system plugin callback load

* fix type

* fix

* fix

* fix: plugin dataset quote;perf: system variables init (#3316)

* fix: plugin dataset quote

* perf: system variables init

* perf: node templates ui;fix: dataset import ui (#3318)

* fix: dataset import ui

* perf: node templates ui

* perf: ui refresh

* feat:套餐改名和套餐跳转配置 (#3309)

* fixing:except Sidebar

* 去除了多余的代码

* 修正了套餐说明的代码

* 修正了误删除的show_git代码

* 修正了名字部分等代码

* 修正了问题,遗留了其他和ui讨论不一致的部分

* 4.8.15 test (#3319)

* remove log

* pref: bill ui

* pref: bill ui

* perf: log

* html渲染文档 (#3270)

* html渲染文档

* 文档有点小问题

* feat: doc (#3322)

* 集合重训练 (#3282)

* rebaser

* 一点补充

* 小问题

* 其他问题修正，删除集合保留文件的参数还没找到...

* reTraining

* delete uesless

* 删除了一行错误代码

* 集合重训练部分

* fixing

* 删除console代码

* feat: navbar item config (#3326)

* perf: custom navbar code;perf: retraining code;feat: api dataset and dataset api doc (#3329)

* feat: api dataset and dataset api doc

* perf: retraining code

* perf: custom navbar code

* fix: ts (#3330)

* fix: ts

* fix: ts

* retraining ui

* perf: api collection filter

* perf: retrining button

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com>
Co-authored-by: papapatrick <109422393+Patrickill@users.noreply.github.com>

projects/app/src/web/core/ai/api.ts

@@ -1,5 +1,5 @@
 import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
-import type { CreateQuestionGuideParams } from '@/global/core/ai/api.d';
+import type { CreateQuestionGuideParams } from '@/pages/api/core/ai/agent/v2/createQuestionGuide';
 
 export const postQuestionGuide = (data: CreateQuestionGuideParams, cancelToken: AbortController) =>
-  POST<string[]>('/core/ai/agent/createQuestionGuide', data, { cancelToken });
+  POST<string[]>('/core/ai/agent/v2/createQuestionGuide', data, { cancelToken });

projects/app/src/web/core/app/api/plugin.ts

@@ -1,4 +1,4 @@
-import { DELETE, GET, POST } from '@/web/common/api/request';
+import { GET, POST } from '@/web/common/api/request';
 import type { createHttpPluginBody } from '@/pages/api/core/app/httpPlugin/create';
 import type { UpdateHttpPluginBody } from '@/pages/api/core/app/httpPlugin/update';
 import type {
@@ -13,6 +13,7 @@ import type { GetPreviewNodeQuery } from '@/pages/api/core/app/plugin/getPreview
 import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
 import { ParentIdType, ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
 import { GetSystemPluginTemplatesBody } from '@/pages/api/core/app/plugin/getSystemPluginTemplates';
+import { PluginGroupSchemaType } from '@fastgpt/service/core/app/plugin/type';
 
 /* ============ team plugin ============== */
 export const getTeamPlugTemplates = (data?: ListAppBody) =>
@@ -40,6 +41,9 @@ export const getTeamPlugTemplates = (data?: ListAppBody) =>
 export const getSystemPlugTemplates = (data: GetSystemPluginTemplatesBody) =>
   POST<NodeTemplateListItemType[]>('/core/app/plugin/getSystemPluginTemplates', data);
 
+export const getPluginGroups = () =>
+  GET<PluginGroupSchemaType[]>('/proApi/core/app/plugin/getPluginGroups');
+
 export const getSystemPluginPaths = (parentId: ParentIdType) => {
   if (!parentId) return Promise.resolve<ParentTreePathItemType[]>([]);
   return GET<ParentTreePathItemType[]>('/core/app/plugin/path', { parentId });

projects/app/src/web/core/dataset/api.ts

@@ -13,12 +13,14 @@ import type {
 import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq.d';
 import type {
   AddTagsToCollectionsParams,
+  ApiDatasetCreateDatasetCollectionParams,
   CreateDatasetCollectionParams,
   CreateDatasetCollectionTagParams,
   CsvTableCreateDatasetCollectionParams,
   DatasetUpdateBody,
   ExternalFileCreateDatasetCollectionParams,
   FileIdCreateDatasetCollectionParams,
+  reTrainingDatasetFileCollectionParams,
   LinkCreateDatasetCollectionParams,
   PostWebsiteSyncParams,
   TextCreateDatasetCollectionParams,
@@ -56,6 +58,14 @@ import type { UpdateDatasetDataProps } from '@fastgpt/global/core/dataset/contro
 import type { DatasetFolderCreateBody } from '@/pages/api/core/dataset/folder/create';
 import type { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
 import type { GetScrollCollectionsProps } from '@/pages/api/core/dataset/collection/scrollList';
+import type {
+  GetApiDatasetFileListProps,
+  GetApiDatasetFileListResponse
+} from '@/pages/api/core/dataset/apiDataset/list';
+import type {
+  listExistIdQuery,
+  listExistIdResponse
+} from '@/pages/api/core/dataset/apiDataset/listExistId';
 
 /* ======================== dataset ======================= */
 export const getDatasets = (data: GetDatasetListBody) =>
@@ -106,6 +116,10 @@ export const postCreateDatasetFileCollection = (data: FileIdCreateDatasetCollect
   POST<{ collectionId: string }>(`/core/dataset/collection/create/fileId`, data, {
     timeout: 360000
   });
+export const postReTrainingDatasetFileCollection = (data: reTrainingDatasetFileCollectionParams) =>
+  POST<{ collectionId: string }>(`/core/dataset/collection/create/reTrainingCollection`, data, {
+    timeout: 360000
+  });
 export const postCreateDatasetLinkCollection = (data: LinkCreateDatasetCollectionParams) =>
   POST<{ collectionId: string }>(`/core/dataset/collection/create/link`, data);
 export const postCreateDatasetTextCollection = (data: TextCreateDatasetCollectionParams) =>
@@ -120,13 +134,19 @@ export const postCreateDatasetExternalFileCollection = (
   POST<{ collectionId: string }>(`/proApi/core/dataset/collection/create/externalFileUrl`, data, {
     timeout: 360000
   });
+export const postCreateDatasetApiDatasetCollection = (
+  data: ApiDatasetCreateDatasetCollectionParams
+) =>
+  POST<{ collectionId: string }>(`/core/dataset/collection/create/apiCollection`, data, {
+    timeout: 360000
+  });
 
 export const putDatasetCollectionById = (data: UpdateDatasetCollectionParams) =>
   POST(`/core/dataset/collection/update`, data);
 export const delDatasetCollectionById = (params: { id: string }) =>
   DELETE(`/core/dataset/collection/delete`, params);
 export const postLinkCollectionSync = (collectionId: string) =>
-  POST<DatasetCollectionSyncResultEnum>(`/core/dataset/collection/sync/link`, {
+  POST<DatasetCollectionSyncResultEnum>(`/core/dataset/collection/sync`, {
     collectionId
   });
 
@@ -199,3 +219,9 @@ export const getPreviewChunks = (data: PostPreviewFilesChunksProps) =>
 /* ================== read source ======================== */
 export const getCollectionSource = (data: readCollectionSourceBody) =>
   POST<readCollectionSourceResponse>('/core/dataset/collection/read', data);
+
+/* ================== apiDataset ======================== */
+export const getApiDatasetFileList = (data: GetApiDatasetFileListProps) =>
+  POST<GetApiDatasetFileListResponse>('/core/dataset/apiDataset/list', data);
+export const getApiDatasetFileListExistId = (data: listExistIdQuery) =>
+  GET<listExistIdResponse>('/core/dataset/apiDataset/listExistId', data);

projects/app/src/web/core/dataset/context/datasetPageContext.tsx

@@ -115,7 +115,13 @@ export const DatasetPageContextProvider = ({
     if (datasetId === data.id) {
       setDatasetDetail((state) => ({
         ...state,
-        ...data
+        ...data,
+        apiServer: data.apiServer
+          ? {
+              baseUrl: data.apiServer.baseUrl,
+              authorization: ''
+            }
+          : undefined
       }));
     }
   };

projects/app/src/web/core/dataset/type.d.ts

@@ -2,6 +2,7 @@ import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { ImportProcessWayEnum } from './constants';
 import { UseFormReturn } from 'react-hook-form';
+import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';
 
 export type ImportSourceItemType = {
   id: string;
@@ -30,6 +31,10 @@ export type ImportSourceItemType = {
   // external file
   externalFileUrl?: string;
   externalFileId?: string;
+
+  // api dataset
+  apiFileId?: string;
+  apiFile?: APIFileItem;
 };
 
 export type ImportSourceParamsType = UseFormReturn<

projects/app/src/web/core/workflow/utils.ts

@@ -22,9 +22,7 @@ import { EditorVariablePickerType } from '@fastgpt/web/components/common/Textare
 import {
   formatEditorVariablePickerIcon,
   getAppChatConfig,
-  getGuideModule,
-  isValidArrayReferenceValue,
-  isValidReferenceValue
+  getGuideModule
 } from '@fastgpt/global/core/workflow/utils';
 import { TFunction } from 'next-i18next';
 import {