feat: markdown extension (#3663)

* feat: markdown extension

* media cros

* rerank test

* default price

* perf: default model

* fix: cannot custom provider

* fix: default model select

* update bg

* perf: default model selector

* fix: usage export

* i18n

* fix: rerank

* update init extension

* perf: ip limit check

* doubao model order

* web default modle

* perf: tts selector

* perf: tts error

* qrcode package

projects/app/src/components/Markdown/codeBlock/Audio.tsx

@@ -0,0 +1,31 @@
+import React, { useEffect } from 'react';
+import { Box } from '@chakra-ui/react';
+import { useMarkdownWidth } from '../hooks';
+
+const AudioBlock = ({ code: audioUrl }: { code: string }) => {
+  const { width, Ref } = useMarkdownWidth();
+
+  useEffect(() => {
+    fetch(audioUrl?.trim(), {
+      mode: 'cors',
+      credentials: 'omit'
+    })
+      .then((response) => response.blob())
+      .then((blob) => {
+        const url = URL.createObjectURL(blob);
+        const audio = document.getElementById('player');
+        audio?.setAttribute('src', url);
+      })
+      .catch((err) => {
+        console.log(err);
+      });
+  }, [audioUrl]);
+
+  return (
+    <Box w={width} ref={Ref}>
+      <audio id="player" controls style={{ width: '100%' }} />
+    </Box>
+  );
+};
+
+export default AudioBlock;

projects/app/src/components/Markdown/codeBlock/Video.tsx

@@ -0,0 +1,31 @@
+import React, { useEffect } from 'react';
+import { Box } from '@chakra-ui/react';
+import { useMarkdownWidth } from '../hooks';
+
+const VideoBlock = ({ code: videoUrl }: { code: string }) => {
+  const { width, Ref } = useMarkdownWidth();
+
+  useEffect(() => {
+    fetch(videoUrl?.trim(), {
+      mode: 'cors',
+      credentials: 'omit'
+    })
+      .then((response) => response.blob())
+      .then((blob) => {
+        const url = URL.createObjectURL(blob);
+        const video = document.getElementById('player');
+        video?.setAttribute('src', url);
+      })
+      .catch((err) => {
+        console.log(err);
+      });
+  }, [videoUrl]);
+
+  return (
+    <Box w={width} ref={Ref}>
+      <video id="player" controls />
+    </Box>
+  );
+};
+
+export default VideoBlock;

projects/app/src/components/Markdown/codeBlock/iframe-html.tsx

@@ -16,7 +16,7 @@ import { useCopyData } from '@fastgpt/web/hooks/useCopyData';
 import { useTranslation } from 'next-i18next';
 import { useMarkdownWidth } from '../hooks';
 import type { IconNameType } from '@fastgpt/web/components/common/Icon/type.d';
-import { codeLight } from '../CodeLight';
+import { codeLight } from './CodeLight';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 
 const StyledButton = ({

projects/app/src/components/Markdown/index.tsx

@@ -13,12 +13,14 @@ import dynamic from 'next/dynamic';
 import { Box } from '@chakra-ui/react';
 import { CodeClassNameEnum } from './utils';
 
-const CodeLight = dynamic(() => import('./CodeLight'), { ssr: false });
+const CodeLight = dynamic(() => import('./codeBlock/CodeLight'), { ssr: false });
 const MermaidCodeBlock = dynamic(() => import('./img/MermaidCodeBlock'), { ssr: false });
 const MdImage = dynamic(() => import('./img/Image'), { ssr: false });
 const EChartsCodeBlock = dynamic(() => import('./img/EChartsCodeBlock'), { ssr: false });
 const IframeCodeBlock = dynamic(() => import('./codeBlock/Iframe'), { ssr: false });
 const IframeHtmlCodeBlock = dynamic(() => import('./codeBlock/iframe-html'), { ssr: false });
+const VideoBlock = dynamic(() => import('./codeBlock/Video'), { ssr: false });
+const AudioBlock = dynamic(() => import('./codeBlock/Audio'), { ssr: false });
 
 const ChatGuide = dynamic(() => import('./chat/Guide'), { ssr: false });
 const QuestionGuide = dynamic(() => import('./chat/QuestionGuide'), { ssr: false });
@@ -139,6 +141,12 @@ function Code(e: any) {
         </IframeHtmlCodeBlock>
       );
     }
+    if (codeType === CodeClassNameEnum.video) {
+      return <VideoBlock code={strChildren} />;
+    }
+    if (codeType === CodeClassNameEnum.audio) {
+      return <AudioBlock code={strChildren} />;
+    }
 
     return (
       <CodeLight className={className} codeBlock={codeBlock} match={match}>

projects/app/src/components/Markdown/utils.ts

@@ -7,7 +7,9 @@ export enum CodeClassNameEnum {
   files = 'files',
   latex = 'latex',
   iframe = 'iframe',
-  html = 'html'
+  html = 'html',
+  video = 'video',
+  audio = 'audio'
 }
 
 function htmlTableToLatex(html: string) {

projects/app/src/components/core/ai/AISettingModal/index.tsx

@@ -80,7 +80,9 @@ const AIChatSettingsModal = ({
   const temperature = watch('temperature');
   const useVision = watch('aiChatVision');
 
-  const selectedModel = getWebLLMModel(model);
+  const selectedModel = useMemo(() => {
+    return getWebLLMModel(model);
+  }, [model]);
   const llmSupportVision = !!selectedModel?.vision;
 
   const tokenLimit = useMemo(() => {
@@ -244,7 +246,7 @@ const AIChatSettingsModal = ({
           </Box>
           <Box flex={'1 0 0'}>
             <InputSlider
-              min={100}
+              min={0}
               max={tokenLimit}
               step={200}
               isDisabled={maxToken === undefined}

projects/app/src/components/core/ai/SettingLLMModel/index.tsx

@@ -7,8 +7,8 @@ import AISettingModal, { AIChatSettingsModalProps } from '@/components/core/ai/A
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 import { useTranslation } from 'next-i18next';
 import MyIcon from '@fastgpt/web/components/common/Icon';
-import { useMount } from 'ahooks';
 import AIModelSelector from '@/components/Select/AIModelSelector';
+import { getWebDefaultModel } from '@/web/common/system/utils';
 
 type Props = {
   llmModelType?: `${LLMModelTypeEnum}`;
@@ -24,7 +24,7 @@ const SettingLLMModel = ({
   ...props
 }: AIChatSettingsModalProps & Props) => {
   const { t } = useTranslation();
-  const { llmModelList, defaultModels } = useSystemStore();
+  const { llmModelList } = useSystemStore();
 
   const model = defaultData.model;
 
@@ -39,16 +39,19 @@ const SettingLLMModel = ({
       }),
     [llmModelList, llmModelType]
   );
+  const defaultModel = useMemo(() => {
+    return getWebDefaultModel(modelList).model;
+  }, [modelList]);
 
   // Set default model
   useEffect(() => {
-    if (!llmModelList.find((item) => item.model === model) && !!defaultModels.llm) {
+    if (!modelList.find((item) => item.model === model) && !!defaultModel) {
       onChange({
         ...defaultData,
-        model: defaultModels.llm.model
+        model: defaultModel
       });
     }
-  }, [model, defaultData, llmModelList, defaultModels.llm, onChange]);
+  }, [modelList, model, defaultModel, onChange]);
 
   const {
     isOpen: isOpenAIChatSetting,

projects/app/src/components/core/app/DatasetParamsModal.tsx

@@ -65,7 +65,7 @@ const DatasetParamsModal = ({
   const theme = useTheme();
   const { toast } = useToast();
   const { teamPlanStatus } = useUserStore();
-  const { reRankModelList, llmModelList } = useSystemStore();
+  const { reRankModelList, llmModelList, defaultModels } = useSystemStore();
   const [refresh, setRefresh] = useState(false);
   const [currentTabType, setCurrentTabType] = useState(SearchSettingTabEnum.searchMode);
 
@@ -82,7 +82,7 @@ const DatasetParamsModal = ({
       searchMode,
       usingReRank: !!usingReRank && teamPlanStatus?.standardConstants?.permissionReRank !== false,
       datasetSearchUsingExtensionQuery,
-      datasetSearchExtensionModel: datasetSearchExtensionModel || chatModelSelectList[0]?.value,
+      datasetSearchExtensionModel: datasetSearchExtensionModel || defaultModels.llm?.model,
       datasetSearchExtensionBg
     }
   });

projects/app/src/components/core/app/TTSSelect.tsx

@@ -1,6 +1,6 @@
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
-import { Box, Button, Flex, ModalBody, useDisclosure, Image } from '@chakra-ui/react';
+import { Box, Button, Flex, ModalBody, useDisclosure, Image, HStack } from '@chakra-ui/react';
 import React, { useCallback, useMemo } from 'react';
 import { useTranslation } from 'next-i18next';
 import { TTSTypeEnum } from '@/web/core/app/constants';
@@ -9,13 +9,15 @@ import { useAudioPlay } from '@/web/common/utils/voice';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import MyModal from '@fastgpt/web/components/common/MyModal';
 import MySlider from '@/components/Slider';
-import MySelect from '@fastgpt/web/components/common/MySelect';
 import { defaultTTSConfig } from '@fastgpt/global/core/app/constants';
 import ChatFunctionTip from './Tip';
 import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
 import MyImage from '@fastgpt/web/components/common/Image/MyImage';
 import { useContextSelector } from 'use-context-selector';
 import { AppContext } from '@/pageComponents/app/detail/context';
+import Avatar from '@fastgpt/web/components/common/Avatar';
+import { getModelProvider } from '@fastgpt/global/core/ai/provider';
+import MultipleRowSelect from '@fastgpt/web/components/common/MySelect/MultipleRowSelect';
 
 const TTSSelect = ({
   value = defaultTTSConfig,
@@ -30,28 +32,57 @@ const TTSSelect = ({
 
   const appId = useContextSelector(AppContext, (v) => v.appId);
 
-  const list = useMemo(
+  const selectorList = useMemo(
     () => [
-      { label: t('common:core.app.tts.Close'), value: TTSTypeEnum.none },
-      { label: t('common:core.app.tts.Web'), value: TTSTypeEnum.web },
-      ...ttsModelList.map((item) => item?.voices || []).flat()
+      { label: t('app:tts_close'), value: TTSTypeEnum.none, children: [] },
+      { label: t('app:tts_browser'), value: TTSTypeEnum.web, children: [] },
+      ...ttsModelList.map((model) => {
+        const providerData = getModelProvider(model.provider);
+        return {
+          label: (
+            <HStack>
+              <Avatar borderRadius={'0'} w={'1.25rem'} src={providerData.avatar} />
+              <Box>{t(providerData.name)}</Box>
+            </HStack>
+          ),
+          value: model.model,
+          children: model.voices.map((voice) => ({
+            label: voice.label,
+            value: voice.value
+          }))
+        };
+      })
     ],
     [ttsModelList, t]
   );
 
   const formatValue = useMemo(() => {
     if (!value || !value.type) {
-      return TTSTypeEnum.none;
+      return [TTSTypeEnum.none, undefined];
     }
     if (value.type === TTSTypeEnum.none || value.type === TTSTypeEnum.web) {
-      return value.type;
+      return [value.type, undefined];
     }
-    return value.voice;
+
+    return [value.model, value.voice];
   }, [value]);
-  const formLabel = useMemo(
-    () => list.find((item) => item.value === formatValue)?.label || t('common:common.UnKnow'),
-    [formatValue, list, t]
-  );
+  const formLabel = useMemo(() => {
+    const provider = selectorList.find((item) => item.value === formatValue[0]) || selectorList[0];
+    const voice = provider.children.find((item) => item.value === formatValue[1]);
+    return (
+      <Box minW={'150px'} maxW={'220px'} className="textEllipsis">
+        {voice ? (
+          <Flex alignItems={'center'}>
+            <Box>{provider.label}</Box>
+            <Box>-</Box>
+            <Box>{voice.label}</Box>
+          </Flex>
+        ) : (
+          provider.label
+        )}
+      </Box>
+    );
+  }, [formatValue, selectorList, t]);
 
   const { playAudioByText, cancelAudio, audioLoading, audioPlaying } = useAudioPlay({
     appId,
@@ -59,21 +90,16 @@ const TTSSelect = ({
   });
 
   const onclickChange = useCallback(
-    (e: string) => {
-      if (e === TTSTypeEnum.none || e === TTSTypeEnum.web) {
-        onChange({ type: e as `${TTSTypeEnum}` });
+    (e: string[]) => {
+      console.log(e, '-=');
+      if (e[0] === TTSTypeEnum.none || e[0] === TTSTypeEnum.web) {
+        onChange({ type: e[0] });
       } else {
-        const audioModel = ttsModelList.find((item) =>
-          item.voices?.find((voice) => voice.value === e)
-        );
-        if (!audioModel) {
-          return;
-        }
         onChange({
           ...value,
           type: TTSTypeEnum.model,
-          model: audioModel.model,
-          voice: e
+          model: e[0],
+          voice: e[1]
         });
       }
     },
@@ -113,7 +139,13 @@ const TTSSelect = ({
         <ModalBody px={[5, 16]} py={[4, 8]}>
           <Flex justifyContent={'space-between'} alignItems={'center'}>
             <FormLabel>{t('common:core.app.tts.Speech model')}</FormLabel>
-            <MySelect w={'220px'} value={formatValue} list={list} onchange={onclickChange} />
+            <MultipleRowSelect
+              rowMinWidth="160px"
+              label={formLabel}
+              value={formatValue}
+              list={selectorList}
+              onSelect={onclickChange}
+            />
           </Flex>
           <Flex mt={8} justifyContent={'space-between'}>
             <FormLabel>{t('common:core.app.tts.Speech speed')}</FormLabel>
@@ -135,7 +167,7 @@ const TTSSelect = ({
               }}
             />
           </Flex>
-          {formatValue !== TTSTypeEnum.none && (
+          {formatValue[0] !== TTSTypeEnum.none && (
             <Flex mt={10} justifyContent={'end'}>
               {audioPlaying ? (
                 <Flex>

projects/app/src/components/core/app/WelcomeTextConfig.tsx

@@ -25,7 +25,7 @@ const WelcomeTextConfig = (props: TextareaProps) => {
         mt={1.5}
         rows={6}
         fontSize={'sm'}
-        bg={'white'}
+        bg={'myGray.50'}
         minW={'384px'}
         placeholder={t('common:core.app.tip.welcomeTextTip')}
         autoHeight

projects/app/src/components/support/wallet/QRCodePayModal.tsx

@@ -1,14 +1,12 @@
 import MyModal from '@fastgpt/web/components/common/MyModal';
-import React, { useCallback, useEffect, useRef } from 'react';
+import React, { useCallback, useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'next-i18next';
 import { Box, ModalBody } from '@chakra-ui/react';
 import { checkBalancePayResult } from '@/web/support/wallet/bill/api';
 import { useToast } from '@fastgpt/web/hooks/useToast';
-import { useRouter } from 'next/router';
 import { getErrText } from '@fastgpt/global/common/error/utils';
 import LightTip from '@fastgpt/web/components/common/LightTip';
-import Script from 'next/script';
-import { getWebReqUrl } from '@fastgpt/web/common/system/utils';
+import QRCode from 'qrcode';
 
 export type QRPayProps = {
   readPrice: number;
@@ -27,21 +25,35 @@ const QRCodePayModal = ({
 }: QRPayProps & { tip?: string; onSuccess?: () => any }) => {
   const { t } = useTranslation();
   const { toast } = useToast();
-  const dom = useRef<HTMLDivElement>(null);
+  const canvasRef = useRef<HTMLDivElement>(null);
 
   const drawCode = useCallback(() => {
-    if (dom.current && window.QRCode && !dom.current.innerHTML) {
-      new window.QRCode(dom.current, {
-        text: codeUrl,
-        width: qrCodeSize,
-        height: qrCodeSize,
-        colorDark: '#000000',
-        colorLight: '#ffffff',
-        correctLevel: window.QRCode.CorrectLevel.H
+    const canvas = document.createElement('canvas');
+    QRCode.toCanvas(canvas, codeUrl, {
+      width: qrCodeSize,
+      margin: 0,
+      color: {
+        dark: '#000000',
+        light: '#ffffff'
+      }
+    })
+      .then(() => {
+        if (canvasRef.current) {
+          canvasRef.current.innerHTML = '';
+          canvasRef.current.appendChild(canvas);
+        } else {
+          drawCode();
+        }
+      })
+      .catch((err) => {
+        console.error('QRCode generation error:', err);
       });
-    }
   }, [codeUrl]);
 
+  useEffect(() => {
+    drawCode();
+  }, [drawCode]);
+
   useEffect(() => {
     let timer: NodeJS.Timeout;
     const check = async () => {
@@ -54,7 +66,7 @@ const QRCodePayModal = ({
               title: res,
               status: 'success'
             });
-            return;
+            return clearTimeout(timer);
           } catch (error) {
             toast({
               title: getErrText(error),
@@ -63,9 +75,7 @@ const QRCodePayModal = ({
           }
         }
       } catch (error) {}
-
-      drawCode();
-
+      clearTimeout(timer);
       timer = setTimeout(check, 2000);
     };
 
@@ -75,23 +85,15 @@ const QRCodePayModal = ({
   }, [billId, drawCode, onSuccess, toast]);
 
   return (
-    <>
-      <Script
-        src={getWebReqUrl('/js/qrcode.min.js')}
-        strategy="lazyOnload"
-        onLoad={drawCode}
-      ></Script>
-
-      <MyModal isOpen title={t('common:user.Pay')} iconSrc="/imgs/modal/pay.svg">
-        <ModalBody textAlign={'center'} pb={10} whiteSpace={'pre-wrap'}>
-          {tip && <LightTip text={tip} mb={8} textAlign={'left'} />}
-          <Box ref={dom} id={'payQRCode'} display={'inline-block'} h={`${qrCodeSize}px`}></Box>
-          <Box mt={5} textAlign={'center'}>
-            {t('common:pay.wechat', { price: readPrice })}
-          </Box>
-        </ModalBody>
-      </MyModal>
-    </>
+    <MyModal isOpen title={t('common:user.Pay')} iconSrc="/imgs/modal/pay.svg">
+      <ModalBody textAlign={'center'} pb={10} whiteSpace={'pre-wrap'}>
+        {tip && <LightTip text={tip} mb={8} textAlign={'left'} />}
+        <Box ref={canvasRef} display={'inline-block'} h={`${qrCodeSize}px`}></Box>
+        <Box mt={5} textAlign={'center'}>
+          {t('common:pay.wechat', { price: readPrice })}
+        </Box>
+      </ModalBody>
+    </MyModal>
   );
 };