V4.8.15 feature (#3331)

* feat: add customize toolkit (#3205) * chaoyang * fix-auth * add toolkit * add order * plugin usage * fix * delete console: * Fix: Fix fullscreen preview top positioning and improve Markdown rendering logic (#3247) * 完成任务：修复全屏预览顶部固定问题，优化 Markdown 渲染逻辑 * 有问题修改 * 问题再修改 * 修正问题 * fix: plugin standalone display issue (#3254) * 4.8.15 test (#3246) * o1 config * perf: system plugin code * 调整系统插件代码。增加html 渲染安全配置。 (#3258) * perf: base64 picker * perf: list app or dataset * perf: plugin config code * 小窗适配等问题 (#3257) * 小窗适配等问题 * git问题 * 小窗剩余问题 * feat: system plugin auth and lock version (#3265) * feat: system plugin auth and lock version * update comment * 4.8.15 test (#3267) * tmp log * perf: login direct * perf: iframe html code * remove log * fix: plugin standalone display (#3277) * refactor: 页面拆分&i18n拆分 (#3281) * refactor: account组件拆成独立页面 * script: 新增i18n json文件创建脚本 * refactor: 页面i18n拆分 * i18n: add en&hant * 4.8.15 test (#3285) * tmp log * remove log * fix: watch avatar refresh * perf: i18n code * fix(plugin): use intro instead of userguide (#3290) * Universal SSO (#3292) * tmp log * remove log * feat: common oauth * readme * perf: sso provider * remove sso code * perf: refresh plugins * feat: add api dataset (#3272) * add api-dataset * fix api-dataset * fix api dataset * fix ts * perf: create collection code (#3301) * tmp log * remove log * perf: i18n change * update version doc * feat: question guide from chatId * perf: create collection code * fix: request api * fix: request api * fix: tts auth and response type (#3303) * perf: md splitter * fix: tts auth and response type * fix: api file dataset (#3307) * perf: api dataset init (#3310) * perf: collection schema * perf: api dataset init * refactor: 团队管理独立页面 (#3302) * ui: 团队管理独立页面 * 代码优化 * fix * perf: sync collection and ui check (#3314) * perf: sync collection * remove script * perf: update api server * perf: api dataset parent * perf: team ui * perf: team 18n * update team ui * perf: ui check * perf: i18n * fix: debug variables & cronjob & system plugin callback load (#3315) * fix: debug variables & cronjob & system plugin callback load * fix type * fix * fix * fix: plugin dataset quote;perf: system variables init (#3316) * fix: plugin dataset quote * perf: system variables init * perf: node templates ui;fix: dataset import ui (#3318) * fix: dataset import ui * perf: node templates ui * perf: ui refresh * feat:套餐改名和套餐跳转配置 (#3309) * fixing:except Sidebar * 去除了多余的代码 * 修正了套餐说明的代码 * 修正了误删除的show_git代码 * 修正了名字部分等代码 * 修正了问题,遗留了其他和ui讨论不一致的部分 * 4.8.15 test (#3319) * remove log * pref: bill ui * pref: bill ui * perf: log * html渲染文档 (#3270) * html渲染文档 * 文档有点小问题 * feat: doc (#3322) * 集合重训练 (#3282) * rebaser * 一点补充 * 小问题 * 其他问题修正，删除集合保留文件的参数还没找到... * reTraining * delete uesless * 删除了一行错误代码 * 集合重训练部分 * fixing * 删除console代码 * feat: navbar item config (#3326) * perf: custom navbar code;perf: retraining code;feat: api dataset and dataset api doc (#3329) * feat: api dataset and dataset api doc * perf: retraining code * perf: custom navbar code * fix: ts (#3330) * fix: ts * fix: ts * retraining ui * perf: api collection filter * perf: retrining button --------- Co-authored-by: heheer <heheer@sealos.io> Co-authored-by: Jiangween <145003935+Jiangween@users.noreply.github.com> Co-authored-by: papapatrick <109422393+Patrickill@users.noreply.github.com>
2024-12-06 10:56:53 +08:00
parent b188544386
commit 1aebe5f185
307 changed files with 7383 additions and 3981 deletions
--- a/projects/app/src/web/common/api/request.ts
+++ b/projects/app/src/web/common/api/request.ts
@@ -163,8 +163,8 @@ function request(
      baseURL: getWebReqUrl('/api'),
      url,
      method,
-      data: ['POST', 'PUT'].includes(method) ? data : null,
-      params: !['POST', 'PUT'].includes(method) ? data : null,
+      data: ['POST', 'PUT'].includes(method) ? data : undefined,
+      params: !['POST', 'PUT'].includes(method) ? data : undefined,
      signal: cancelToken?.signal ?? controller?.signal,
      withCredentials,
      ...config // 用户自定义配置，可以覆盖前面的配置
--- a/projects/app/src/web/common/utils/i18n.ts
+++ b/projects/app/src/web/common/utils/i18n.ts
@@ -23,5 +23,5 @@ export const langMap = {

 export const serviceSideProps = (content: any, ns: I18nNsType = []) => {
  const lang = content.req?.cookies?.NEXT_LOCALE || content.locale;
-  return serverSideTranslations(lang, ['common', 'error', ...ns], null, content.locales);
+  return serverSideTranslations(lang, ['common', ...ns], null);
 };
--- a/projects/app/src/web/common/utils/voice.ts
+++ b/projects/app/src/web/common/utils/voice.ts
@@ -1,6 +1,6 @@
 'use client';

-import { useState, useCallback, useEffect, useMemo, useRef } from 'react';
+import { useState, useCallback, useMemo, useRef } from 'react';
 import { useToast } from '@fastgpt/web/hooks/useToast';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import type { AppTTSConfigType } from '@fastgpt/global/core/app/type.d';
@@ -10,17 +10,19 @@ import type { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/ch
 import { useMount } from 'ahooks';
 import { getWebReqUrl } from '@fastgpt/web/common/system/utils';

-const contentType = 'audio/mpeg';
 const splitMarker = 'SPLIT_MARKER';
+const contentType = 'audio/mpeg';

 // 添加 MediaSource 支持检测函数
 const isMediaSourceSupported = () => {
  return typeof MediaSource !== 'undefined' && MediaSource.isTypeSupported?.(contentType);
 };

-export const useAudioPlay = (props?: OutLinkChatAuthProps & { ttsConfig?: AppTTSConfigType }) => {
+export const useAudioPlay = (
+  props?: OutLinkChatAuthProps & { appId: string; ttsConfig?: AppTTSConfigType }
+) => {
  const { t } = useTranslation();
-  const { ttsConfig, shareId, outLinkUid, teamId, teamToken } = props || {};
+  const { appId, ttsConfig, shareId, outLinkUid, teamId, teamToken } = props || {};
  const { toast } = useToast();
  const audioRef = useRef<HTMLAudioElement>();
  const [audioLoading, setAudioLoading] = useState(false);
@@ -53,6 +55,7 @@ export const useAudioPlay = (props?: OutLinkChatAuthProps & { ttsConfig?: AppTTS
        },
        signal: audioController.current.signal,
        body: JSON.stringify({
+          appId,
          ttsConfig,
          input: input.trim(),
          shareId,
@@ -74,7 +77,7 @@ export const useAudioPlay = (props?: OutLinkChatAuthProps & { ttsConfig?: AppTTS
      }
      return response.body;
    },
-    [outLinkUid, shareId, t, teamId, teamToken, toast, ttsConfig]
+    [appId, outLinkUid, shareId, t, teamId, teamToken, toast, ttsConfig]
  );
  const playWebAudio = useCallback((text: string) => {
    // window speech
@@ -118,6 +121,7 @@ export const useAudioPlay = (props?: OutLinkChatAuthProps & { ttsConfig?: AppTTS
        audioRef.current.src = audioUrl;
        audioRef.current.play();
      };
+
      const readAudioStream = (stream: ReadableStream<Uint8Array>) => {
        if (!audioRef.current) return;

@@ -272,7 +276,7 @@ export const useAudioPlay = (props?: OutLinkChatAuthProps & { ttsConfig?: AppTTS
    });
    const sourceBuffer = ms.addSourceBuffer(contentType);
    segmentedSourceBuffer.current = sourceBuffer;
-  }, [cancelAudio, t, toast]);
+  }, [cancelAudio]);
  const finishSegmentedAudio = useCallback(() => {
    if (!isMediaSourceSupported()) {
      // 不支持 MediaSource 时，不需要特殊处理
@@ -391,7 +395,7 @@ export const useAudioPlay = (props?: OutLinkChatAuthProps & { ttsConfig?: AppTTS
        playWebAudio(text);
      }
    },
-    [appendAudioStream, playWebAudio, ttsConfig?.model, ttsConfig?.type]
+    [appendAudioStream, getAudioStream, playWebAudio, ttsConfig?.model, ttsConfig?.type]
  );

  // listen audio status
--- a/projects/app/src/web/core/ai/api.ts
+++ b/projects/app/src/web/core/ai/api.ts
@@ -1,5 +1,5 @@
 import { GET, POST, PUT, DELETE } from '@/web/common/api/request';
-import type { CreateQuestionGuideParams } from '@/global/core/ai/api.d';
+import type { CreateQuestionGuideParams } from '@/pages/api/core/ai/agent/v2/createQuestionGuide';

 export const postQuestionGuide = (data: CreateQuestionGuideParams, cancelToken: AbortController) =>
-  POST<string[]>('/core/ai/agent/createQuestionGuide', data, { cancelToken });
+  POST<string[]>('/core/ai/agent/v2/createQuestionGuide', data, { cancelToken });
--- a/projects/app/src/web/core/app/api/plugin.ts
+++ b/projects/app/src/web/core/app/api/plugin.ts
@@ -1,4 +1,4 @@
-import { DELETE, GET, POST } from '@/web/common/api/request';
+import { GET, POST } from '@/web/common/api/request';
 import type { createHttpPluginBody } from '@/pages/api/core/app/httpPlugin/create';
 import type { UpdateHttpPluginBody } from '@/pages/api/core/app/httpPlugin/update';
 import type {
@@ -13,6 +13,7 @@ import type { GetPreviewNodeQuery } from '@/pages/api/core/app/plugin/getPreview
 import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
 import { ParentIdType, ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
 import { GetSystemPluginTemplatesBody } from '@/pages/api/core/app/plugin/getSystemPluginTemplates';
+import { PluginGroupSchemaType } from '@fastgpt/service/core/app/plugin/type';

 /* ============ team plugin ============== */
 export const getTeamPlugTemplates = (data?: ListAppBody) =>
@@ -40,6 +41,9 @@ export const getTeamPlugTemplates = (data?: ListAppBody) =>
 export const getSystemPlugTemplates = (data: GetSystemPluginTemplatesBody) =>
  POST<NodeTemplateListItemType[]>('/core/app/plugin/getSystemPluginTemplates', data);

+export const getPluginGroups = () =>
+  GET<PluginGroupSchemaType[]>('/proApi/core/app/plugin/getPluginGroups');
+
 export const getSystemPluginPaths = (parentId: ParentIdType) => {
  if (!parentId) return Promise.resolve<ParentTreePathItemType[]>([]);
  return GET<ParentTreePathItemType[]>('/core/app/plugin/path', { parentId });
--- a/projects/app/src/web/core/dataset/api.ts
+++ b/projects/app/src/web/core/dataset/api.ts
@@ -13,12 +13,14 @@ import type {
 import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq.d';
 import type {
  AddTagsToCollectionsParams,
+  ApiDatasetCreateDatasetCollectionParams,
  CreateDatasetCollectionParams,
  CreateDatasetCollectionTagParams,
  CsvTableCreateDatasetCollectionParams,
  DatasetUpdateBody,
  ExternalFileCreateDatasetCollectionParams,
  FileIdCreateDatasetCollectionParams,
+  reTrainingDatasetFileCollectionParams,
  LinkCreateDatasetCollectionParams,
  PostWebsiteSyncParams,
  TextCreateDatasetCollectionParams,
@@ -56,6 +58,14 @@ import type { UpdateDatasetDataProps } from '@fastgpt/global/core/dataset/contro
 import type { DatasetFolderCreateBody } from '@/pages/api/core/dataset/folder/create';
 import type { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
 import type { GetScrollCollectionsProps } from '@/pages/api/core/dataset/collection/scrollList';
+import type {
+  GetApiDatasetFileListProps,
+  GetApiDatasetFileListResponse
+} from '@/pages/api/core/dataset/apiDataset/list';
+import type {
+  listExistIdQuery,
+  listExistIdResponse
+} from '@/pages/api/core/dataset/apiDataset/listExistId';

 /* ======================== dataset ======================= */
 export const getDatasets = (data: GetDatasetListBody) =>
@@ -106,6 +116,10 @@ export const postCreateDatasetFileCollection = (data: FileIdCreateDatasetCollect
  POST<{ collectionId: string }>(`/core/dataset/collection/create/fileId`, data, {
    timeout: 360000
  });
+export const postReTrainingDatasetFileCollection = (data: reTrainingDatasetFileCollectionParams) =>
+  POST<{ collectionId: string }>(`/core/dataset/collection/create/reTrainingCollection`, data, {
+    timeout: 360000
+  });
 export const postCreateDatasetLinkCollection = (data: LinkCreateDatasetCollectionParams) =>
  POST<{ collectionId: string }>(`/core/dataset/collection/create/link`, data);
 export const postCreateDatasetTextCollection = (data: TextCreateDatasetCollectionParams) =>
@@ -120,13 +134,19 @@ export const postCreateDatasetExternalFileCollection = (
  POST<{ collectionId: string }>(`/proApi/core/dataset/collection/create/externalFileUrl`, data, {
    timeout: 360000
  });
+export const postCreateDatasetApiDatasetCollection = (
+  data: ApiDatasetCreateDatasetCollectionParams
+) =>
+  POST<{ collectionId: string }>(`/core/dataset/collection/create/apiCollection`, data, {
+    timeout: 360000
+  });

 export const putDatasetCollectionById = (data: UpdateDatasetCollectionParams) =>
  POST(`/core/dataset/collection/update`, data);
 export const delDatasetCollectionById = (params: { id: string }) =>
  DELETE(`/core/dataset/collection/delete`, params);
 export const postLinkCollectionSync = (collectionId: string) =>
-  POST<DatasetCollectionSyncResultEnum>(`/core/dataset/collection/sync/link`, {
+  POST<DatasetCollectionSyncResultEnum>(`/core/dataset/collection/sync`, {
    collectionId
  });

@@ -199,3 +219,9 @@ export const getPreviewChunks = (data: PostPreviewFilesChunksProps) =>
 /* ================== read source ======================== */
 export const getCollectionSource = (data: readCollectionSourceBody) =>
  POST<readCollectionSourceResponse>('/core/dataset/collection/read', data);
+
+/* ================== apiDataset ======================== */
+export const getApiDatasetFileList = (data: GetApiDatasetFileListProps) =>
+  POST<GetApiDatasetFileListResponse>('/core/dataset/apiDataset/list', data);
+export const getApiDatasetFileListExistId = (data: listExistIdQuery) =>
+  GET<listExistIdResponse>('/core/dataset/apiDataset/listExistId', data);
--- a/projects/app/src/web/core/dataset/context/datasetPageContext.tsx
+++ b/projects/app/src/web/core/dataset/context/datasetPageContext.tsx
@@ -115,7 +115,13 @@ export const DatasetPageContextProvider = ({
    if (datasetId === data.id) {
      setDatasetDetail((state) => ({
        ...state,
-        ...data
+        ...data,
+        apiServer: data.apiServer
+          ? {
+              baseUrl: data.apiServer.baseUrl,
+              authorization: ''
+            }
+          : undefined
      }));
    }
  };
--- a/projects/app/src/web/core/dataset/type.d.ts
+++ b/projects/app/src/web/core/dataset/type.d.ts
@@ -2,6 +2,7 @@ import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api
 import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
 import { ImportProcessWayEnum } from './constants';
 import { UseFormReturn } from 'react-hook-form';
+import { APIFileItem } from '@fastgpt/global/core/dataset/apiDataset';

 export type ImportSourceItemType = {
  id: string;
@@ -30,6 +31,10 @@ export type ImportSourceItemType = {
  // external file
  externalFileUrl?: string;
  externalFileId?: string;
+
+  // api dataset
+  apiFileId?: string;
+  apiFile?: APIFileItem;
 };

 export type ImportSourceParamsType = UseFormReturn<
--- a/projects/app/src/web/core/workflow/utils.ts
+++ b/projects/app/src/web/core/workflow/utils.ts
@@ -22,9 +22,7 @@ import { EditorVariablePickerType } from '@fastgpt/web/components/common/Textare
 import {
  formatEditorVariablePickerIcon,
  getAppChatConfig,
-  getGuideModule,
-  isValidArrayReferenceValue,
-  isValidReferenceValue
+  getGuideModule
 } from '@fastgpt/global/core/workflow/utils';
 import { TFunction } from 'next-i18next';
 import {
--- a/projects/app/src/web/support/wallet/sub/constants.ts
+++ b/projects/app/src/web/support/wallet/sub/constants.ts
@@ -1,2 +1,18 @@
+import { getDocPath } from '@/web/common/system/doc';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+
 export const AI_POINT_USAGE_CARD_ROUTE = '/price#point-card';
+export const getAiPointUsageCardRoute = () => {
+  const subPlans = useSystemStore.getState().subPlans;
+  return subPlans?.planDescriptionUrl
+    ? getDocPath(subPlans.planDescriptionUrl)
+    : AI_POINT_USAGE_CARD_ROUTE;
+};
+
 export const EXTRA_PLAN_CARD_ROUTE = '/price#extra-plan';
+export const getExtraPlanCardRoute = () => {
+  const subPlans = useSystemStore.getState().subPlans;
+  return subPlans?.planDescriptionUrl
+    ? getDocPath(subPlans.planDescriptionUrl)
+    : EXTRA_PLAN_CARD_ROUTE;
+};