V4.9.4 feature (#4470)

* Training status (#4424)

* dataset data training state (#4311)

* dataset data training state

* fix

* fix ts

* fix

* fix api format

* fix

* fix

* perf: count training

* format

* fix: dataset training state (#4417)

* fix

* add test

* fix

* fix

* fix test

* fix test

* perf: training count

* count

* loading status

---------

Co-authored-by: heheer <heheer@sealos.io>

* doc

* website sync feature (#4429)

* perf: introduce BullMQ for website sync (#4403)

* perf: introduce BullMQ for website sync

* feat: new redis module

* fix: remove graceful shutdown

* perf: improve UI in dataset detail

- Updated the "change" icon SVG file.
- Modified i18n strings.
- Added new i18n string "immediate_sync".
- Improved UI in dataset detail page, including button icons and
background colors.

* refactor: Add chunkSettings to DatasetSchema

* perf: website sync ux

* env template

* fix: clean up website dataset when updating chunk settings (#4420)

* perf: check setting updated

* perf: worker currency

* feat: init script for website sync refactor (#4425)

* website feature doc

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

* pro migration (#4388) (#4433)

* pro migration

* reuse customPdfParseType

Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>

* perf: remove loading ui

* feat: config chat file expired time

* Redis cache (#4436)

* perf: add Redis cache for vector counting (#4432)

* feat: cache

* perf: get cache key

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

* perf: mobile voice input (#4437)

* update:Mobile voice interaction (#4362)

* Add files via upload

* Add files via upload

* Update ollama.md

* Update ollama.md

* Add files via upload

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update constants.ts

* Add files via upload

* Update ChatInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update useSpeech.ts

* Update ChatInput.tsx

* Add files via upload

* Update common.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update common.json

* Update common.json

* Update common.json

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update VoiceInput.tsx

* speech ui

* 优化语音输入组件，调整输入框显示逻辑，修复语音输入遮罩层样式，更新画布背景透明度，增强用户交互体验。 (#4435)

* perf: mobil voice input

---------

Co-authored-by: dreamer6680 <1468683855@qq.com>

* Test completion v2 (#4438)

* add v2 completions (#4364)

* add v2 completions

* completion config

* config version

* fix

* frontend

* doc

* fix

* fix: completions v2 api

---------

Co-authored-by: heheer <heheer@sealos.io>

* package

* Test mongo log (#4443)

* feat: mongodb-log (#4426)

* perf: mongo log

* feat: completions stop reasoner

* mongo db log

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>

* update doc

* Update doc

* fix external var ui (#4444)

* action

* fix: ts (#4458)

* preview doc action

add docs preview permission

update preview action

udpate action

* update doc (#4460)

* update preview action

* update doc

* remove

* update

* schema

* update mq export;perf: redis cache  (#4465)

* perf: redis cache

* update mq export

* perf: website sync error tip

* add error worker

* website sync ui (#4466)

* Updated the dynamic display of the voice input pop-up (#4469)

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* fix: voice input

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>
Co-authored-by: dreamer6680 <1468683855@qq.com>
Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>

projects/app/src/components/core/ai/SettingLLMModel/index.tsx

@@ -99,7 +99,6 @@ const SettingLLMModel = ({
         <AISettingModal
           onClose={onCloseAIChatSetting}
           onSuccess={(e) => {
-            console.log(e);
             onChange(e);
             onCloseAIChatSetting();
           }}

projects/app/src/components/core/chat/ChatContainer/ChatBox/Input/ChatInput.tsx

@@ -1,7 +1,6 @@
-import { useSpeech } from '@/web/common/hooks/useSpeech';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { Box, Flex, Spinner, Textarea } from '@chakra-ui/react';
-import React, { useRef, useEffect, useCallback, useMemo } from 'react';
+import React, { useRef, useEffect, useCallback, useMemo, useState } from 'react';
 import { useTranslation } from 'next-i18next';
 import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 import MyIcon from '@fastgpt/web/components/common/Icon';
@@ -18,6 +17,7 @@ import FilePreview from '../../components/FilePreview';
 import { useFileUpload } from '../hooks/useFileUpload';
 import ComplianceTip from '@/components/common/ComplianceTip/index';
 import { useToast } from '@fastgpt/web/hooks/useToast';
+import VoiceInput, { type VoiceInputComponentRef } from './VoiceInput';
 
 const InputGuideBox = dynamic(() => import('./InputGuideBox'));
 
@@ -44,6 +44,7 @@ const ChatInput = ({
   const { t } = useTranslation();
   const { toast } = useToast();
   const { isPc } = useSystem();
+  const VoiceInputRef = useRef<VoiceInputComponentRef>(null);
 
   const { setValue, watch, control } = chatForm;
   const inputValue = watch('input');
@@ -53,7 +54,6 @@ const ChatInput = ({
   const chatId = useContextSelector(ChatBoxContext, (v) => v.chatId);
   const isChatting = useContextSelector(ChatBoxContext, (v) => v.isChatting);
   const whisperConfig = useContextSelector(ChatBoxContext, (v) => v.whisperConfig);
-  const autoTTSResponse = useContextSelector(ChatBoxContext, (v) => v.autoTTSResponse);
   const chatInputGuide = useContextSelector(ChatBoxContext, (v) => v.chatInputGuide);
   const fileSelectConfig = useContextSelector(ChatBoxContext, (v) => v.fileSelectConfig);
 
@@ -106,86 +106,6 @@ const ChatInput = ({
     [TextareaDom, canSendMessage, fileList, onSendMessage, replaceFiles]
   );
 
-  /* whisper init */
-  const canvasRef = useRef<HTMLCanvasElement>(null);
-  const {
-    isSpeaking,
-    isTransCription,
-    stopSpeak,
-    startSpeak,
-    speakingTimeString,
-    renderAudioGraph,
-    stream
-  } = useSpeech({ appId, ...outLinkAuthData });
-  const onWhisperRecord = useCallback(() => {
-    const finishWhisperTranscription = (text: string) => {
-      if (!text) return;
-      if (whisperConfig?.autoSend) {
-        onSendMessage({
-          text,
-          files: fileList,
-          autoTTSResponse
-        });
-        replaceFiles([]);
-      } else {
-        resetInputVal({ text });
-      }
-    };
-    if (isSpeaking) {
-      return stopSpeak();
-    }
-    startSpeak(finishWhisperTranscription);
-  }, [
-    autoTTSResponse,
-    fileList,
-    isSpeaking,
-    onSendMessage,
-    replaceFiles,
-    resetInputVal,
-    startSpeak,
-    stopSpeak,
-    whisperConfig?.autoSend
-  ]);
-  useEffect(() => {
-    if (!stream) {
-      return;
-    }
-    const audioContext = new AudioContext();
-    const analyser = audioContext.createAnalyser();
-    analyser.fftSize = 4096;
-    analyser.smoothingTimeConstant = 1;
-    const source = audioContext.createMediaStreamSource(stream);
-    source.connect(analyser);
-    const renderCurve = () => {
-      if (!canvasRef.current) return;
-      renderAudioGraph(analyser, canvasRef.current);
-      window.requestAnimationFrame(renderCurve);
-    };
-    renderCurve();
-  }, [renderAudioGraph, stream]);
-
-  const RenderTranslateLoading = useMemo(
-    () => (
-      <Flex
-        position={'absolute'}
-        top={0}
-        bottom={0}
-        left={0}
-        right={0}
-        zIndex={10}
-        pl={5}
-        alignItems={'center'}
-        bg={'white'}
-        color={'primary.500'}
-        visibility={isSpeaking && isTransCription ? 'visible' : 'hidden'}
-      >
-        <Spinner size={'sm'} mr={4} />
-        {t('common:core.chat.Converting to text')}
-      </Flex>
-    ),
-    [isSpeaking, isTransCription, t]
-  );
-
   const RenderTextarea = useMemo(
     () => (
       <Flex alignItems={'flex-end'} mt={fileList.length > 0 ? 1 : 0} pl={[2, 4]}>
@@ -198,7 +118,6 @@ const ChatInput = ({
             cursor={'pointer'}
             transform={'translateY(1px)'}
             onClick={() => {
-              if (isSpeaking) return;
               onOpenSelectFile();
             }}
           >
@@ -208,7 +127,6 @@ const ChatInput = ({
             <File onSelect={(files) => onSelectFile({ files })} />
           </Flex>
         )}
-
         {/* input area */}
         <Textarea
           ref={TextareaDom}
@@ -220,11 +138,7 @@ const ChatInput = ({
             border: 'none'
           }}
           placeholder={
-            isSpeaking
-              ? t('common:core.chat.Speaking')
-              : isPc
-                ? t('common:core.chat.Type a message')
-                : t('chat:input_placeholder_phone')
+            isPc ? t('common:core.chat.Type a message') : t('chat:input_placeholder_phone')
           }
           resize={'none'}
           rows={1}
@@ -237,9 +151,8 @@ const ChatInput = ({
           wordBreak={'break-all'}
           boxShadow={'none !important'}
           color={'myGray.900'}
-          isDisabled={isSpeaking}
-          value={inputValue}
           fontSize={['md', 'sm']}
+          value={inputValue}
           onChange={(e) => {
             const textarea = e.target;
             textarea.style.height = textareaMinH;
@@ -290,118 +203,78 @@ const ChatInput = ({
             }
           }}
         />
-        <Flex alignItems={'center'} position={'absolute'} right={[2, 4]} bottom={['10px', '12px']}>
-          {/* voice-input */}
-          {whisperConfig?.open && !inputValue && !isChatting && (
-            <>
-              <canvas
-                ref={canvasRef}
-                style={{
-                  height: '30px',
-                  width: isSpeaking && !isTransCription ? '100px' : 0,
-                  background: 'white',
-                  zIndex: 0
+        <Flex
+          alignItems={'center'}
+          position={'absolute'}
+          right={[2, 4]}
+          bottom={['10px', '12px']}
+          zIndex={3}
+        >
+          {/* Voice input icon */}
+          {whisperConfig?.open && !inputValue && (
+            <MyTooltip label={t('common:core.chat.Record')}>
+              <Flex
+                alignItems={'center'}
+                justifyContent={'center'}
+                flexShrink={0}
+                h={['28px', '32px']}
+                w={['28px', '32px']}
+                mr={2}
+                borderRadius={'md'}
+                cursor={'pointer'}
+                _hover={{ bg: '#F5F5F8' }}
+                onClick={() => {
+                  VoiceInputRef.current?.onSpeak?.();
                 }}
-              />
-              {isSpeaking && (
-                <MyTooltip label={t('common:core.chat.Cancel Speak')}>
-                  <Flex
-                    mr={2}
-                    alignItems={'center'}
-                    justifyContent={'center'}
-                    flexShrink={0}
-                    h={['26px', '32px']}
-                    w={['26px', '32px']}
-                    borderRadius={'md'}
-                    cursor={'pointer'}
-                    _hover={{ bg: '#F5F5F8' }}
-                    onClick={() => stopSpeak(true)}
-                  >
-                    <MyIcon
-                      name={'core/chat/cancelSpeak'}
-                      width={['20px', '22px']}
-                      height={['20px', '22px']}
-                    />
-                  </Flex>
-                </MyTooltip>
-              )}
-              <MyTooltip
-                label={
-                  isSpeaking ? t('common:core.chat.Finish Speak') : t('common:core.chat.Record')
-                }
               >
-                <Flex
-                  mr={2}
-                  alignItems={'center'}
-                  justifyContent={'center'}
-                  flexShrink={0}
-                  h={['26px', '32px']}
-                  w={['26px', '32px']}
-                  borderRadius={'md'}
-                  cursor={'pointer'}
-                  _hover={{ bg: '#F5F5F8' }}
-                  onClick={onWhisperRecord}
-                >
-                  <MyIcon
-                    name={isSpeaking ? 'core/chat/finishSpeak' : 'core/chat/recordFill'}
-                    width={['20px', '22px']}
-                    height={['20px', '22px']}
-                    color={isSpeaking ? 'primary.500' : 'myGray.600'}
-                  />
-                </Flex>
-              </MyTooltip>
-            </>
-          )}
-          {/* send and stop icon */}
-          {isSpeaking ? (
-            <Box color={'#5A646E'} w={'36px'} textAlign={'right'} whiteSpace={'nowrap'}>
-              {speakingTimeString}
-            </Box>
-          ) : (
-            <Flex
-              alignItems={'center'}
-              justifyContent={'center'}
-              flexShrink={0}
-              h={['28px', '32px']}
-              w={['28px', '32px']}
-              borderRadius={'md'}
-              bg={
-                isSpeaking || isChatting
-                  ? ''
-                  : !havInput || hasFileUploading
-                    ? '#E5E5E5'
-                    : 'primary.500'
-              }
-              cursor={havInput ? 'pointer' : 'not-allowed'}
-              lineHeight={1}
-              onClick={() => {
-                if (isChatting) {
-                  return onStop();
-                }
-                return handleSend();
-              }}
-            >
-              {isChatting ? (
                 <MyIcon
-                  animation={'zoomStopIcon 0.4s infinite alternate'}
+                  name={'core/chat/recordFill'}
                   width={['22px', '25px']}
                   height={['22px', '25px']}
-                  cursor={'pointer'}
-                  name={'stop'}
-                  color={'gray.500'}
+                  color={'myGray.600'}
                 />
-              ) : (
-                <MyTooltip label={t('common:core.chat.Send Message')}>
-                  <MyIcon
-                    name={'core/chat/sendFill'}
-                    width={['18px', '20px']}
-                    height={['18px', '20px']}
-                    color={'white'}
-                  />
-                </MyTooltip>
-              )}
-            </Flex>
+              </Flex>
+            </MyTooltip>
           )}
+
+          {/* send and stop icon */}
+          <Flex
+            alignItems={'center'}
+            justifyContent={'center'}
+            flexShrink={0}
+            h={['28px', '32px']}
+            w={['28px', '32px']}
+            borderRadius={'md'}
+            bg={isChatting ? '' : !havInput || hasFileUploading ? '#E5E5E5' : 'primary.500'}
+            cursor={havInput ? 'pointer' : 'not-allowed'}
+            lineHeight={1}
+            onClick={() => {
+              if (isChatting) {
+                return onStop();
+              }
+              return handleSend();
+            }}
+          >
+            {isChatting ? (
+              <MyIcon
+                animation={'zoomStopIcon 0.4s infinite alternate'}
+                width={['22px', '25px']}
+                height={['22px', '25px']}
+                cursor={'pointer'}
+                name={'stop'}
+                color={'gray.500'}
+              />
+            ) : (
+              <MyTooltip label={t('common:core.chat.Send Message')}>
+                <MyIcon
+                  name={'core/chat/sendFill'}
+                  width={['18px', '20px']}
+                  height={['18px', '20px']}
+                  color={'white'}
+                />
+              </MyTooltip>
+            )}
+          </Flex>
         </Flex>
       </Flex>
     ),
@@ -415,21 +288,15 @@ const ChatInput = ({
       inputValue,
       isChatting,
       isPc,
-      isSpeaking,
-      isTransCription,
       onOpenSelectFile,
       onSelectFile,
       onStop,
-      onWhisperRecord,
       selectFileIcon,
       selectFileLabel,
       setValue,
       showSelectFile,
       showSelectImg,
-      speakingTimeString,
-      stopSpeak,
-      t,
-      whisperConfig?.open
+      t
     ]
   );
 
@@ -468,7 +335,7 @@ const ChatInput = ({
         pt={fileList.length > 0 ? '0' : ['14px', '18px']}
         pb={['14px', '18px']}
         position={'relative'}
-        boxShadow={isSpeaking ? `0 0 10px rgba(54,111,255,0.4)` : `0 0 10px rgba(0,0,0,0.2)`}
+        boxShadow={`0 0 10px rgba(0,0,0,0.2)`}
         borderRadius={['none', 'md']}
         bg={'white'}
         overflow={'display'}
@@ -495,15 +362,20 @@ const ChatInput = ({
             }}
           />
         )}
-
-        {/* translate loading */}
-        {RenderTranslateLoading}
-
         {/* file preview */}
         <Box px={[1, 3]}>
           <FilePreview fileList={fileList} removeFiles={removeFiles} />
         </Box>
 
+        {/* voice input and loading container */}
+        {!inputValue && (
+          <VoiceInput
+            ref={VoiceInputRef}
+            onSendMessage={onSendMessage}
+            resetInputVal={resetInputVal}
+          />
+        )}
+
         {RenderTextarea}
       </Box>
       <ComplianceTip type={'chat'} />

projects/app/src/components/core/chat/ChatContainer/ChatBox/Input/VoiceInput.tsx

@@ -0,0 +1,369 @@
+import { useSpeech } from '@/web/common/hooks/useSpeech';
+import { Box, Flex, HStack, Spinner } from '@chakra-ui/react';
+import React, {
+  useRef,
+  useEffect,
+  useCallback,
+  useState,
+  forwardRef,
+  useImperativeHandle,
+  useMemo
+} from 'react';
+import { useTranslation } from 'next-i18next';
+import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import { useSystem } from '@fastgpt/web/hooks/useSystem';
+import { useContextSelector } from 'use-context-selector';
+import { ChatBoxContext } from '../Provider';
+import MyIconButton from '@/pageComponents/account/team/OrgManage/IconButton';
+
+export interface VoiceInputComponentRef {
+  onSpeak: () => void;
+}
+
+type VoiceInputProps = {
+  onSendMessage: (params: { text: string; files?: any[]; autoTTSResponse?: boolean }) => void;
+  resetInputVal: (val: { text: string }) => void;
+};
+
+// PC voice input
+const PCVoiceInput = ({
+  speakingTimeString,
+  stopSpeak,
+  canvasRef
+}: {
+  speakingTimeString: string;
+  stopSpeak: (param: boolean) => void;
+  canvasRef: React.RefObject<HTMLCanvasElement>;
+}) => {
+  const { t } = useTranslation();
+
+  return (
+    <HStack h={'100%'} px={4}>
+      <Box fontSize="sm" color="myGray.500" flex={'1 0 0'}>
+        {t('common:core.chat.Speaking')}
+      </Box>
+      <canvas
+        ref={canvasRef}
+        style={{
+          height: '10px',
+          width: '100px',
+          background: 'white'
+        }}
+      />
+      <Box fontSize="sm" color="myGray.500" whiteSpace={'nowrap'}>
+        {speakingTimeString}
+      </Box>
+      <MyTooltip label={t('common:core.chat.Cancel Speak')}>
+        <MyIconButton
+          name={'core/chat/cancelSpeak'}
+          h={'22px'}
+          w={'22px'}
+          onClick={() => stopSpeak(true)}
+        />
+      </MyTooltip>
+      <MyTooltip label={t('common:core.chat.Finish Speak')}>
+        <MyIconButton
+          name={'core/chat/finishSpeak'}
+          h={'22px'}
+          w={'22px'}
+          onClick={() => stopSpeak(false)}
+        />
+      </MyTooltip>
+    </HStack>
+  );
+};
+
+// mobile voice input
+const MobileVoiceInput = ({
+  isSpeaking,
+  onStartSpeak,
+  onCloseSpeak,
+  stopSpeak,
+  canvasRef
+}: {
+  isSpeaking: boolean;
+  onStartSpeak: () => void;
+  onCloseSpeak: () => any;
+  stopSpeak: (param: boolean) => void;
+  canvasRef: React.RefObject<HTMLCanvasElement>;
+}) => {
+  const { t } = useTranslation();
+
+  const isPressing = useRef(false);
+  const startTimeRef = useRef(0); // 防抖
+
+  const startYRef = useRef(0);
+
+  const [isCancel, setIsCancel] = useState(false);
+  const canvasPosition = canvasRef.current?.getBoundingClientRect();
+  const maskBottom = canvasPosition ? `${window.innerHeight - canvasPosition.top}px` : '50px';
+
+  const handleTouchStart = useCallback(
+    (e: React.TouchEvent<HTMLDivElement>) => {
+      isPressing.current = true;
+      setIsCancel(false);
+
+      startTimeRef.current = Date.now();
+      const touch = e.touches[0];
+      startYRef.current = touch.pageY;
+
+      onStartSpeak();
+    },
+    [onStartSpeak]
+  );
+
+  const handleTouchMove = useCallback(
+    (e: React.TouchEvent<HTMLDivElement>) => {
+      const touch = e.touches[0] as Touch;
+      const currentY = touch.pageY;
+      const deltaY = startYRef.current - currentY;
+
+      if (deltaY > 90) {
+        setIsCancel(true);
+      } else if (deltaY <= 90) {
+        setIsCancel(false);
+      }
+    },
+    [startYRef]
+  );
+
+  const handleTouchEnd = useCallback(
+    (e: React.TouchEvent<HTMLDivElement>) => {
+      if (!isPressing.current) return;
+
+      const endTime = Date.now();
+      const timeDifference = endTime - startTimeRef.current;
+
+      if (isCancel || timeDifference < 200) {
+        stopSpeak(true);
+      } else {
+        stopSpeak(false);
+      }
+    },
+    [isCancel, stopSpeak]
+  );
+
+  return (
+    <Flex position="relative" h="100%">
+      {/* Back Icon */}
+      {!isSpeaking && (
+        <MyTooltip label={t('chat:back_to_text')}>
+          <MyIconButton
+            position="absolute"
+            right={2}
+            top={'50%'}
+            transform={'translateY(-50%)'}
+            zIndex={5}
+            name={'core/chat/backText'}
+            h={'22px'}
+            w={'22px'}
+            onClick={onCloseSpeak}
+          />
+        </MyTooltip>
+      )}
+      <Flex
+        alignItems={'center'}
+        justifyContent={'center'}
+        h="100%"
+        flex="1 0 0"
+        bg={isSpeaking ? (isCancel ? 'red.500' : 'primary.500') : 'white'}
+        onTouchMove={handleTouchMove}
+        onTouchEnd={handleTouchEnd}
+        onTouchStart={handleTouchStart}
+        onTouchCancel={() => {
+          stopSpeak(true);
+        }}
+        zIndex={4}
+      >
+        <Box visibility={isSpeaking ? 'hidden' : 'visible'}>{t('chat:press_to_speak')}</Box>
+        <Box
+          position="absolute"
+          h={'100%'}
+          w={'100%'}
+          as="canvas"
+          ref={canvasRef}
+          flex="0 0 80%"
+          visibility={isSpeaking ? 'visible' : 'hidden'}
+        />
+      </Flex>
+
+      {/* Mask */}
+      {isSpeaking && (
+        <Flex
+          justifyContent="center"
+          alignItems="center"
+          height="100%"
+          position="fixed"
+          left={0}
+          right={0}
+          bottom={maskBottom}
+          h={'200px'}
+          bg="linear-gradient(to top, white, rgba(255, 255, 255, 0.7), rgba(255, 255, 255, 0))"
+        >
+          <Box fontSize="sm" color="myGray.500" position="absolute" bottom={'10px'}>
+            {isCancel ? t('chat:release_cancel') : t('chat:release_send')}
+          </Box>
+        </Flex>
+      )}
+    </Flex>
+  );
+};
+
+const VoiceInput = forwardRef<VoiceInputComponentRef, VoiceInputProps>(
+  ({ onSendMessage, resetInputVal }, ref) => {
+    const { t } = useTranslation();
+    const { isPc } = useSystem();
+
+    const outLinkAuthData = useContextSelector(ChatBoxContext, (v) => v.outLinkAuthData);
+    const appId = useContextSelector(ChatBoxContext, (v) => v.appId);
+    const whisperConfig = useContextSelector(ChatBoxContext, (v) => v.whisperConfig);
+    const autoTTSResponse = useContextSelector(ChatBoxContext, (v) => v.autoTTSResponse);
+    const canvasRef = useRef<HTMLCanvasElement>(null);
+
+    const {
+      isSpeaking,
+      isTransCription,
+      stopSpeak,
+      startSpeak,
+      speakingTimeString,
+      renderAudioGraphPc,
+      renderAudioGraphMobile,
+      stream
+    } = useSpeech({ appId, ...outLinkAuthData });
+
+    const [mobilePreSpeak, setMobilePreSpeak] = useState(false);
+
+    // Canvas render
+    useEffect(() => {
+      if (!stream) {
+        return;
+      }
+
+      const audioContext = new AudioContext();
+      const analyser = audioContext.createAnalyser();
+      analyser.fftSize = 4096;
+      analyser.smoothingTimeConstant = 1;
+      const source = audioContext.createMediaStreamSource(stream);
+      source.connect(analyser);
+
+      let animationFrameId: number | null = null;
+      const renderCurve = () => {
+        const canvas = canvasRef.current;
+        if (!canvas) return;
+
+        const ctx = canvas.getContext('2d');
+        if (!ctx) return;
+
+        if (!stream.active) {
+          ctx.clearRect(0, 0, canvas.width, canvas.height);
+          if (animationFrameId) {
+            window.cancelAnimationFrame(animationFrameId);
+            animationFrameId = null;
+          }
+          return;
+        }
+
+        if (isPc) {
+          renderAudioGraphPc(analyser, canvas);
+        } else {
+          renderAudioGraphMobile(analyser, canvas);
+        }
+        animationFrameId = window.requestAnimationFrame(renderCurve);
+      };
+
+      renderCurve();
+
+      return () => {
+        if (animationFrameId) {
+          window.cancelAnimationFrame(animationFrameId);
+        }
+        audioContext.close();
+        source.disconnect();
+        analyser.disconnect();
+      };
+    }, [stream, canvasRef, renderAudioGraphPc, renderAudioGraphMobile, isPc]);
+
+    const onStartSpeak = useCallback(() => {
+      const finishWhisperTranscription = (text: string) => {
+        if (!text) return;
+        if (whisperConfig?.autoSend) {
+          onSendMessage({
+            text,
+            autoTTSResponse
+          });
+        } else {
+          resetInputVal({ text });
+        }
+      };
+      startSpeak(finishWhisperTranscription);
+    }, [autoTTSResponse, onSendMessage, resetInputVal, startSpeak, whisperConfig?.autoSend]);
+
+    const onSpeach = useCallback(() => {
+      if (isPc) {
+        onStartSpeak();
+      } else {
+        setMobilePreSpeak(true);
+      }
+    }, [isPc, onStartSpeak]);
+    useImperativeHandle(ref, () => ({
+      onSpeak: onSpeach
+    }));
+
+    if (!whisperConfig?.open) return null;
+    if (!mobilePreSpeak && !isSpeaking && !isTransCription) return null;
+
+    return (
+      <Box
+        position="absolute"
+        overflow={'hidden'}
+        userSelect={'none'}
+        top={0}
+        left={0}
+        right={0}
+        bottom={0}
+        bg="white"
+        zIndex={5}
+        borderRadius={isPc ? 'md' : ''}
+        onContextMenu={(e) => e.preventDefault()}
+      >
+        {isPc ? (
+          <PCVoiceInput
+            speakingTimeString={speakingTimeString}
+            stopSpeak={stopSpeak}
+            canvasRef={canvasRef}
+          />
+        ) : (
+          <MobileVoiceInput
+            isSpeaking={isSpeaking}
+            onStartSpeak={onStartSpeak}
+            onCloseSpeak={() => setMobilePreSpeak(false)}
+            stopSpeak={stopSpeak}
+            canvasRef={canvasRef}
+          />
+        )}
+
+        {isTransCription && (
+          <Flex
+            position={'absolute'}
+            top={0}
+            bottom={0}
+            left={0}
+            right={0}
+            pl={5}
+            alignItems={'center'}
+            bg={'white'}
+            color={'primary.500'}
+            zIndex={6}
+          >
+            <Spinner size={'sm'} mr={4} />
+            {t('common:core.chat.Converting to text')}
+          </Flex>
+        )}
+      </Box>
+    );
+  }
+);
+VoiceInput.displayName = 'VoiceInput';
+
+export default VoiceInput;

projects/app/src/components/core/chat/ChatContainer/ChatBox/index.tsx

@@ -219,7 +219,8 @@ const ChatBox = ({
       tool,
       interactive,
       autoTTSResponse,
-      variables
+      variables,
+      nodeResponse
     }: generatingMessageProps & { autoTTSResponse?: boolean }) => {
       setChatRecords((state) =>
         state.map((item, index) => {
@@ -232,7 +233,14 @@ const ChatBox = ({
             JSON.stringify(item.value[item.value.length - 1])
           );
 
-          if (event === SseResponseEventEnum.flowNodeStatus && status) {
+          if (event === SseResponseEventEnum.flowNodeResponse && nodeResponse) {
+            return {
+              ...item,
+              responseData: item.responseData
+                ? [...item.responseData, nodeResponse]
+                : [nodeResponse]
+            };
+          } else if (event === SseResponseEventEnum.flowNodeStatus && status) {
             return {
               ...item,
               status,
@@ -518,36 +526,34 @@ const ChatBox = ({
               reserveTool: true
             });
 
-            const {
-              responseData,
-              responseText,
-              isNewChat = false
-            } = await onStartChat({
+            const { responseText } = await onStartChat({
               messages, // 保证最后一条是 Human 的消息
               responseChatItemId: responseChatId,
               controller: abortSignal,
               generatingMessage: (e) => generatingMessage({ ...e, autoTTSResponse }),
               variables: requestVariables
             });
-            if (responseData?.[responseData.length - 1]?.error) {
-              toast({
-                title: t(responseData[responseData.length - 1].error?.message),
-                status: 'error'
-              });
-            }
 
             // Set last chat finish status
             let newChatHistories: ChatSiteItemType[] = [];
             setChatRecords((state) => {
               newChatHistories = state.map((item, index) => {
                 if (index !== state.length - 1) return item;
+
+                // Check node response error
+                const responseData = mergeChatResponseData(item.responseData || []);
+                if (responseData[responseData.length - 1]?.error) {
+                  toast({
+                    title: t(responseData[responseData.length - 1].error?.message),
+                    status: 'error'
+                  });
+                }
+
                 return {
                   ...item,
                   status: ChatStatusEnum.finish,
                   time: new Date(),
-                  responseData: item.responseData
-                    ? mergeChatResponseData([...item.responseData, ...responseData])
-                    : responseData
+                  responseData
                 };
               });
               return newChatHistories;
@@ -567,7 +573,7 @@ const ChatBox = ({
           } catch (err: any) {
             console.log(err);
             toast({
-              title: t(getErrText(err, 'core.chat.error.Chat error') as any),
+              title: t(getErrText(err, t('common:core.chat.error.Chat error') as any)),
               status: 'error',
               duration: 5000,
               isClosable: true
@@ -807,12 +813,14 @@ const ChatBox = ({
       showEmptyIntro &&
       chatRecords.length === 0 &&
       !variableList?.length &&
+      !externalVariableList?.length &&
       !welcomeText,
     [
       chatRecords.length,
       feConfigs?.show_emptyChat,
       showEmptyIntro,
       variableList?.length,
+      externalVariableList?.length,
       welcomeText
     ]
   );

projects/app/src/components/core/chat/ChatContainer/PluginRunBox/context.tsx

@@ -18,6 +18,7 @@ import { ChatItemContext } from '@/web/core/chat/context/chatItemContext';
 import { ChatRecordContext } from '@/web/core/chat/context/chatRecordContext';
 import { AppFileSelectConfigType } from '@fastgpt/global/core/app/type';
 import { defaultAppSelectFileConfig } from '@fastgpt/global/core/app/constants';
+import { mergeChatResponseData } from '@fastgpt/global/core/chat/utils';
 
 type PluginRunContextType = PluginRunBoxProps & {
   isChatting: boolean;
@@ -46,11 +47,12 @@ const PluginRunContextProvider = ({
 
   const pluginInputs = useContextSelector(ChatItemContext, (v) => v.chatBoxData?.app?.pluginInputs);
   const setTab = useContextSelector(ChatItemContext, (v) => v.setPluginRunTab);
+  const variablesForm = useContextSelector(ChatItemContext, (v) => v.variablesForm);
+  const chatConfig = useContextSelector(ChatItemContext, (v) => v.chatBoxData?.app?.chatConfig);
+
   const setChatRecords = useContextSelector(ChatRecordContext, (v) => v.setChatRecords);
   const chatRecords = useContextSelector(ChatRecordContext, (v) => v.chatRecords);
 
-  const chatConfig = useContextSelector(ChatItemContext, (v) => v.chatBoxData?.app?.chatConfig);
-
   const { instruction = '', fileSelectConfig = defaultAppSelectFileConfig } = useMemo(
     () => chatConfig || {},
     [chatConfig]
@@ -65,7 +67,7 @@ const PluginRunContextProvider = ({
   }, []);
 
   const generatingMessage = useCallback(
-    ({ event, text = '', status, name, tool }: generatingMessageProps) => {
+    ({ event, text = '', status, name, tool, nodeResponse, variables }: generatingMessageProps) => {
       setChatRecords((state) =>
         state.map((item, index) => {
           if (index !== state.length - 1 || item.obj !== ChatRoleEnum.AI) return item;
@@ -74,7 +76,14 @@ const PluginRunContextProvider = ({
             JSON.stringify(item.value[item.value.length - 1])
           );
 
-          if (event === SseResponseEventEnum.flowNodeStatus && status) {
+          if (event === SseResponseEventEnum.flowNodeResponse && nodeResponse) {
+            return {
+              ...item,
+              responseData: item.responseData
+                ? [...item.responseData, nodeResponse]
+                : [nodeResponse]
+            };
+          } else if (event === SseResponseEventEnum.flowNodeStatus && status) {
             return {
               ...item,
               status,
@@ -144,13 +153,15 @@ const PluginRunContextProvider = ({
                 return val;
               })
             };
+          } else if (event === SseResponseEventEnum.updateVariables && variables) {
+            variablesForm.setValue('variables', variables);
           }
 
           return item;
         })
       );
     },
-    [setChatRecords]
+    [setChatRecords, variablesForm]
   );
 
   const isChatting = useMemo(
@@ -226,7 +237,7 @@ const PluginRunContextProvider = ({
           }
         }
 
-        const { responseData } = await onStartChat({
+        await onStartChat({
           messages,
           controller: chatController.current,
           generatingMessage,
@@ -235,16 +246,20 @@ const PluginRunContextProvider = ({
             ...formatVariables
           }
         });
-        if (responseData?.[responseData.length - 1]?.error) {
-          toast({
-            title: responseData[responseData.length - 1].error?.message,
-            status: 'error'
-          });
-        }
 
         setChatRecords((state) =>
           state.map((item, index) => {
             if (index !== state.length - 1) return item;
+
+            // Check node response error
+            const responseData = mergeChatResponseData(item.responseData || []);
+            if (responseData[responseData.length - 1]?.error) {
+              toast({
+                title: t(responseData[responseData.length - 1].error?.message),
+                status: 'error'
+              });
+            }
+
             return {
               ...item,
               status: 'finish',

projects/app/src/components/core/chat/ChatContainer/type.d.ts

@@ -1,6 +1,10 @@
 import { StreamResponseType } from '@/web/common/api/fetch';
 import { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type';
-import { ChatSiteItemType, ToolModuleResponseItemType } from '@fastgpt/global/core/chat/type';
+import {
+  ChatHistoryItemResType,
+  ChatSiteItemType,
+  ToolModuleResponseItemType
+} from '@fastgpt/global/core/chat/type';
 import { WorkflowInteractiveResponseType } from '@fastgpt/global/core/workflow/template/system/interactive/type';
 
 export type generatingMessageProps = {
@@ -12,6 +16,7 @@ export type generatingMessageProps = {
   tool?: ToolModuleResponseItemType;
   interactive?: WorkflowInteractiveResponseType;
   variables?: Record<string, any>;
+  nodeResponse?: ChatHistoryItemResType;
 };
 
 export type StartChatFnProps = {

projects/app/src/components/core/chat/components/WholeResponseModal.tsx

@@ -17,6 +17,7 @@ import { ChatBoxContext } from '../ChatContainer/ChatBox/Provider';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { getFileIcon } from '@fastgpt/global/common/file/icon';
 import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
+import { completionFinishReasonMap } from '@fastgpt/global/core/ai/constants';
 
 type sideTabItemType = {
   moduleLogo?: string;
@@ -196,6 +197,13 @@ export const WholeResponseContent = ({
           label={t('common:core.chat.response.module maxToken')}
           value={activeModule?.maxToken}
         />
+        {activeModule?.finishReason && (
+          <Row
+            label={t('chat:completion_finish_reason')}
+            value={t(completionFinishReasonMap[activeModule?.finishReason])}
+          />
+        )}
+
         <Row label={t('chat:reasoning_text')} value={activeModule?.reasoningText} />
         <Row
           label={t('common:core.chat.response.module historyPreview')}

projects/app/src/global/core/dataset/type.d.ts

@@ -29,6 +29,7 @@ export type DatasetCollectionsListItemType = {
 
   dataAmount: number;
   trainingAmount: number;
+  hasError?: boolean;
 };
 
 /* ================= data ===================== */

projects/app/src/instrumentation.ts

@@ -1,6 +1,6 @@
 import { exit } from 'process';
 
-/* 
+/*
   Init system
 */
 export async function register() {
@@ -9,6 +9,7 @@ export async function register() {
       // 基础系统初始化
       const [
         { connectMongo },
+        { connectionMongo, connectionLogMongo, MONGO_URL, MONGO_LOG_URL },
         { systemStartCb },
         { initGlobalVariables, getInitConfig, initSystemPluginGroups, initAppTemplateTypes },
         { initVectorStore },
@@ -19,6 +20,7 @@ export async function register() {
         { startTrainingQueue }
       ] = await Promise.all([
         import('@fastgpt/service/common/mongo/init'),
+        import('@fastgpt/service/common/mongo/index'),
         import('@fastgpt/service/common/system/tools'),
         import('@/service/common/system'),
         import('@fastgpt/service/common/vectorStore/controller'),
@@ -34,7 +36,8 @@ export async function register() {
       initGlobalVariables();
 
       // Connect to MongoDB
-      await connectMongo();
+      await connectMongo(connectionMongo, MONGO_URL);
+      connectMongo(connectionLogMongo, MONGO_LOG_URL);
 
       //init system config；init vector database；init root user
       await Promise.all([getInitConfig(), initVectorStore(), initRootUser()]);

projects/app/src/pageComponents/app/detail/WorkflowComponents/Flow/nodes/NodePluginIO/InputTypeConfig.tsx

@@ -297,7 +297,7 @@ const InputTypeConfig = ({
             <FormLabel flex={'0 0 132px'} fontWeight={'medium'}>
               {t('common:core.module.Default Value')}
             </FormLabel>
-            <Flex alignItems={'center'} flex={1} h={10}>
+            <Flex flex={1} h={10}>
               {(inputType === FlowNodeInputTypeEnum.numberInput ||
                 (inputType === VariableInputEnum.custom &&
                   valueType === WorkflowIOValueTypeEnum.number)) && (

projects/app/src/pageComponents/app/detail/useChatTest.tsx

@@ -48,7 +48,7 @@ export const useChatTest = ({
       const histories = messages.slice(-1);
 
       // 流请求，获取数据
-      const { responseText, responseData } = await streamFetch({
+      const { responseText } = await streamFetch({
         url: '/api/core/chat/chatTest',
         data: {
           // Send histories and user messages
@@ -66,7 +66,7 @@ export const useChatTest = ({
         abortCtrl: controller
       });
 
-      return { responseText, responseData };
+      return { responseText };
     }
   );
 

projects/app/src/pageComponents/chat/ChatHistorySlider.tsx

@@ -84,7 +84,6 @@ const ChatHistorySlider = ({ confirmClearText }: { confirmClearText: string }) =
 
   return (
     <MyBox
-      isLoading={isLoading}
       display={'flex'}
       flexDirection={'column'}
       w={'100%'}

projects/app/src/pageComponents/dataset/detail/CollectionCard/Context.tsx

@@ -1,19 +1,18 @@
 import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
-import { Dispatch, ReactNode, SetStateAction, useEffect, useState } from 'react';
+import { Dispatch, ReactNode, SetStateAction, useState } from 'react';
 import { useTranslation } from 'next-i18next';
 import { createContext, useContextSelector } from 'use-context-selector';
-import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
-import { useRequest } from '@fastgpt/web/hooks/useRequest';
-import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { useDisclosure } from '@chakra-ui/react';
 import { checkTeamWebSyncLimit } from '@/web/support/user/team/api';
-import { postCreateTrainingUsage } from '@/web/support/wallet/usage/api';
 import { getDatasetCollections, postWebsiteSync } from '@/web/core/dataset/api';
 import dynamic from 'next/dynamic';
 import { usePagination } from '@fastgpt/web/hooks/usePagination';
 import { DatasetCollectionsListItemType } from '@/global/core/dataset/type';
 import { useRouter } from 'next/router';
 import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import { WebsiteConfigFormType } from './WebsiteConfig';
 
 const WebSiteConfigModal = dynamic(() => import('./WebsiteConfig'));
 
@@ -66,7 +65,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
   const router = useRouter();
   const { parentId = '' } = router.query as { parentId: string };
 
-  const { datasetDetail, datasetId, updateDataset } = useContextSelector(
+  const { datasetDetail, datasetId, updateDataset, loadDatasetDetail } = useContextSelector(
     DatasetPageContext,
     (v) => v
   );
@@ -75,30 +74,32 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
   const { openConfirm: openWebSyncConfirm, ConfirmModal: ConfirmWebSyncModal } = useConfirm({
     content: t('dataset:start_sync_website_tip')
   });
+  const syncWebsite = async () => {
+    await checkTeamWebSyncLimit();
+    postWebsiteSync({ datasetId: datasetId }).then(() => {
+      loadDatasetDetail(datasetId);
+    });
+  };
   const {
     isOpen: isOpenWebsiteModal,
     onOpen: onOpenWebsiteModal,
     onClose: onCloseWebsiteModal
   } = useDisclosure();
-  const { mutate: onUpdateDatasetWebsiteConfig } = useRequest({
-    mutationFn: async (websiteConfig: DatasetSchemaType['websiteConfig']) => {
-      onCloseWebsiteModal();
-      await checkTeamWebSyncLimit();
+  const { runAsync: onUpdateDatasetWebsiteConfig } = useRequest2(
+    async (websiteConfig: WebsiteConfigFormType) => {
       await updateDataset({
         id: datasetId,
-        websiteConfig,
-        status: DatasetStatusEnum.syncing
+        websiteConfig: websiteConfig.websiteConfig,
+        chunkSettings: websiteConfig.chunkSettings
       });
-      const billId = await postCreateTrainingUsage({
-        name: t('common:core.dataset.training.Website Sync'),
-        datasetId: datasetId
-      });
-      await postWebsiteSync({ datasetId: datasetId, billId });
-
-      return;
+      await syncWebsite();
     },
-    errorToast: t('common:common.Update Failed')
-  });
+    {
+      onSuccess() {
+        onCloseWebsiteModal();
+      }
+    }
+  );
 
   // collection list
   const [searchText, setSearchText] = useState('');
@@ -124,7 +125,7 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
   });
 
   const contextValue: CollectionPageContextType = {
-    openWebSyncConfirm: openWebSyncConfirm(onUpdateDatasetWebsiteConfig),
+    openWebSyncConfirm: openWebSyncConfirm(syncWebsite),
     onOpenWebsiteModal,
 
     searchText,
@@ -149,10 +150,6 @@ const CollectionPageContextProvider = ({ children }: { children: ReactNode }) =>
             <WebSiteConfigModal
               onClose={onCloseWebsiteModal}
               onSuccess={onUpdateDatasetWebsiteConfig}
-              defaultValue={{
-                url: datasetDetail?.websiteConfig?.url,
-                selector: datasetDetail?.websiteConfig?.selector
-              }}
             />
           )}
           <ConfirmWebSyncModal />

projects/app/src/pageComponents/dataset/detail/CollectionCard/EmptyCollectionTip.tsx

@@ -25,6 +25,9 @@ const EmptyCollectionTip = () => {
               {datasetDetail.status === DatasetStatusEnum.syncing && (
                 <>{t('common:core.dataset.status.syncing')}</>
               )}
+              {datasetDetail.status === DatasetStatusEnum.waiting && (
+                <>{t('common:core.dataset.status.waiting')}</>
+              )}
               {datasetDetail.status === DatasetStatusEnum.active && (
                 <>
                   {!datasetDetail?.websiteConfig?.url ? (

projects/app/src/pageComponents/dataset/detail/CollectionCard/Header.tsx

@@ -1,35 +1,23 @@
 import React from 'react';
-import {
-  Box,
-  Flex,
-  MenuButton,
-  Button,
-  Link,
-  useTheme,
-  useDisclosure,
-  HStack
-} from '@chakra-ui/react';
+import { Box, Flex, MenuButton, Button, Link, useDisclosure, HStack } from '@chakra-ui/react';
 import {
   getDatasetCollectionPathById,
   postDatasetCollection,
   putDatasetCollectionById
 } from '@/web/core/dataset/api';
-import { useQuery } from '@tanstack/react-query';
 import { useTranslation } from 'next-i18next';
 import MyIcon from '@fastgpt/web/components/common/Icon';
 import MyInput from '@/components/MyInput';
-import { useRequest, useRequest2 } from '@fastgpt/web/hooks/useRequest';
+import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
 import { useRouter } from 'next/router';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
 import MyMenu from '@fastgpt/web/components/common/MyMenu';
 import { useEditTitle } from '@/web/common/hooks/useEditTitle';
 import {
   DatasetCollectionTypeEnum,
-  TrainingModeEnum,
   DatasetTypeEnum,
   DatasetTypeMap,
-  DatasetStatusEnum,
-  DatasetCollectionDataProcessModeEnum
+  DatasetStatusEnum
 } from '@fastgpt/global/core/dataset/constants';
 import EditFolderModal, { useEditFolder } from '../../EditFolderModal';
 import { TabEnum } from '../../../../pages/dataset/detail/index';
@@ -43,26 +31,36 @@ import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContex
 import { useSystem } from '@fastgpt/web/hooks/useSystem';
 import HeaderTagPopOver from './HeaderTagPopOver';
 import MyBox from '@fastgpt/web/components/common/MyBox';
+import Icon from '@fastgpt/web/components/common/Icon';
+import MyTag from '@fastgpt/web/components/common/Tag/index';
+import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 
 const FileSourceSelector = dynamic(() => import('../Import/components/FileSourceSelector'));
 
-const Header = ({}: {}) => {
+const Header = ({ hasTrainingData }: { hasTrainingData: boolean }) => {
   const { t } = useTranslation();
-  const theme = useTheme();
-
   const { feConfigs } = useSystemStore();
+  const { isPc } = useSystem();
+
   const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
 
   const router = useRouter();
   const { parentId = '' } = router.query as { parentId: string };
-  const { isPc } = useSystem();
 
-  const { searchText, setSearchText, total, getData, pageNum, onOpenWebsiteModal } =
-    useContextSelector(CollectionPageContext, (v) => v);
+  const {
+    searchText,
+    setSearchText,
+    total,
+    getData,
+    pageNum,
+    onOpenWebsiteModal,
+    openWebSyncConfirm
+  } = useContextSelector(CollectionPageContext, (v) => v);
 
-  const { data: paths = [] } = useQuery(['getDatasetCollectionPathById', parentId], () =>
-    getDatasetCollectionPathById(parentId)
-  );
+  const { data: paths = [] } = useRequest2(() => getDatasetCollectionPathById(parentId), {
+    refreshDeps: [parentId],
+    manual: false
+  });
 
   const { editFolderData, setEditFolderData } = useEditFolder();
   const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } =
@@ -72,13 +70,14 @@ const Header = ({}: {}) => {
       canEmpty: false
     });
 
+  // Import collection
   const {
     isOpen: isOpenFileSourceSelector,
     onOpen: onOpenFileSourceSelector,
     onClose: onCloseFileSourceSelector
   } = useDisclosure();
 
-  const { runAsync: onCreateCollection, loading: onCreating } = useRequest2(
+  const { runAsync: onCreateCollection } = useRequest2(
     async ({ name, type }: { name: string; type: DatasetCollectionTypeEnum }) => {
       const id = await postDatasetCollection({
         parentId,
@@ -100,7 +99,7 @@ const Header = ({}: {}) => {
   const isWebSite = datasetDetail?.type === DatasetTypeEnum.websiteDataset;
 
   return (
-    <MyBox isLoading={onCreating} display={['block', 'flex']} alignItems={'center'} gap={2}>
+    <MyBox display={['block', 'flex']} alignItems={'center'} gap={2}>
       <HStack flex={1}>
         <Box flex={1} fontWeight={'500'} color={'myGray.900'} whiteSpace={'nowrap'}>
           <ParentPath
@@ -121,13 +120,15 @@ const Header = ({}: {}) => {
                   {!isWebSite && <MyIcon name="common/list" mr={2} w={'20px'} color={'black'} />}
                   {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel as any)}({total})
                 </Flex>
+                {/* Website sync */}
                 {datasetDetail?.websiteConfig?.url && (
                   <Flex fontSize={'mini'}>
-                    {t('common:core.dataset.website.Base Url')}:
+                    <Box>{t('common:core.dataset.website.Base Url')}:</Box>
                     <Link
+                      className="textEllipsis"
+                      maxW={'300px'}
                       href={datasetDetail.websiteConfig.url}
                       target="_blank"
-                      mr={2}
                       color={'blue.700'}
                     >
                       {datasetDetail.websiteConfig.url}
@@ -171,12 +172,14 @@ const Header = ({}: {}) => {
         )}
 
         {/* Tag */}
-        {datasetDetail.permission.hasWritePer && feConfigs?.isPlus && <HeaderTagPopOver />}
+        {datasetDetail.type !== DatasetTypeEnum.websiteDataset &&
+          datasetDetail.permission.hasWritePer &&
+          feConfigs?.isPlus && <HeaderTagPopOver />}
       </HStack>
 
       {/* diff collection button */}
       {datasetDetail.permission.hasWritePer && (
-        <Box textAlign={'end'} mt={[3, 0]}>
+        <Box mt={[3, 0]}>
           {datasetDetail?.type === DatasetTypeEnum.dataset && (
             <MyMenu
               offset={[0, 5]}
@@ -233,9 +236,8 @@ const Header = ({}: {}) => {
                       onClick: () => {
                         onOpenCreateVirtualFileModal({
                           defaultVal: '',
-                          onSuccess: (name) => {
-                            onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual });
-                          }
+                          onSuccess: (name) =>
+                            onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual })
                         });
                       }
                     },
@@ -272,35 +274,70 @@ const Header = ({}: {}) => {
           {datasetDetail?.type === DatasetTypeEnum.websiteDataset && (
             <>
               {datasetDetail?.websiteConfig?.url ? (
-                <Flex alignItems={'center'}>
+                <>
                   {datasetDetail.status === DatasetStatusEnum.active && (
-                    <Button onClick={onOpenWebsiteModal}>{t('common:common.Config')}</Button>
+                    <HStack gap={2}>
+                      <Button
+                        onClick={onOpenWebsiteModal}
+                        leftIcon={<Icon name="change" w={'1rem'} />}
+                      >
+                        {t('dataset:params_config')}
+                      </Button>
+                      {!hasTrainingData && (
+                        <Button
+                          variant={'whitePrimary'}
+                          onClick={openWebSyncConfirm}
+                          leftIcon={<Icon name="common/confirm/restoreTip" w={'1rem'} />}
+                        >
+                          {t('dataset:immediate_sync')}
+                        </Button>
+                      )}
+                    </HStack>
                   )}
                   {datasetDetail.status === DatasetStatusEnum.syncing && (
-                    <Flex
-                      ml={3}
-                      alignItems={'center'}
+                    <MyTag
+                      colorSchema="purple"
+                      showDot
                       px={3}
-                      py={1}
-                      borderRadius="md"
-                      border={theme.borders.base}
+                      h={'36px'}
+                      DotStyles={{
+                        w: '8px',
+                        h: '8px',
+                        animation: 'zoomStopIcon 0.5s infinite alternate'
+                      }}
                     >
-                      <Box
-                        animation={'zoomStopIcon 0.5s infinite alternate'}
-                        bg={'myGray.700'}
-                        w="8px"
-                        h="8px"
-                        borderRadius={'50%'}
-                        mt={'1px'}
-                      ></Box>
-                      <Box ml={2} color={'myGray.600'}>
-                        {t('common:core.dataset.status.syncing')}
-                      </Box>
-                    </Flex>
+                      {t('common:core.dataset.status.syncing')}
+                    </MyTag>
                   )}
-                </Flex>
+                  {datasetDetail.status === DatasetStatusEnum.waiting && (
+                    <MyTag
+                      colorSchema="gray"
+                      showDot
+                      px={3}
+                      h={'36px'}
+                      DotStyles={{
+                        w: '8px',
+                        h: '8px',
+                        animation: 'zoomStopIcon 0.5s infinite alternate'
+                      }}
+                    >
+                      {t('common:core.dataset.status.waiting')}
+                    </MyTag>
+                  )}
+                  {datasetDetail.status === DatasetStatusEnum.error && (
+                    <MyTag colorSchema="red" showDot px={3} h={'36px'}>
+                      <HStack spacing={1}>
+                        <Box>{t('dataset:status_error')}</Box>
+                        <QuestionTip color={'red.500'} label={datasetDetail.errorMsg} />
+                      </HStack>
+                    </MyTag>
+                  )}
+                </>
               ) : (
-                <Button onClick={onOpenWebsiteModal}>
+                <Button
+                  onClick={onOpenWebsiteModal}
+                  leftIcon={<Icon name="common/setting" w={'18px'} />}
+                >
                   {t('common:core.dataset.Set Website Config')}
                 </Button>
               )}

projects/app/src/pageComponents/dataset/detail/CollectionCard/TrainingStates.tsx

@@ -0,0 +1,502 @@
+import {
+  Box,
+  Button,
+  Flex,
+  ModalBody,
+  Table,
+  TableContainer,
+  Tbody,
+  Td,
+  Th,
+  Thead,
+  Tr
+} from '@chakra-ui/react';
+import MyModal from '@fastgpt/web/components/common/MyModal';
+import { useTranslation } from 'next-i18next';
+import MyTag from '@fastgpt/web/components/common/Tag/index';
+import FillRowTabs from '@fastgpt/web/components/common/Tabs/FillRowTabs';
+import { useMemo, useState } from 'react';
+import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
+import {
+  deleteTrainingData,
+  getDatasetCollectionTrainingDetail,
+  getTrainingDataDetail,
+  getTrainingError,
+  updateTrainingData
+} from '@/web/core/dataset/api';
+import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
+import { getTrainingDataDetailResponse } from '@/pages/api/core/dataset/training/getTrainingDataDetail';
+import MyTextarea from '@/components/common/Textarea/MyTextarea';
+import { TrainingProcess } from '@/web/core/dataset/constants';
+import { useForm } from 'react-hook-form';
+import type { getTrainingDetailResponse } from '@/pages/api/core/dataset/collection/trainingDetail';
+import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
+import EmptyTip from '@fastgpt/web/components/common/EmptyTip';
+
+enum TrainingStatus {
+  NotStart = 'NotStart',
+  Queued = 'Queued', // wait count>0
+  Running = 'Running', // wait count=0; training count>0.
+  Ready = 'Ready',
+  Error = 'Error'
+}
+
+const ProgressView = ({ trainingDetail }: { trainingDetail: getTrainingDetailResponse }) => {
+  const { t } = useTranslation();
+
+  const isQA = trainingDetail?.trainingType === DatasetCollectionDataProcessModeEnum.qa;
+
+  /* 
+    状态计算
+    1. 暂时没有内容解析的状态
+    2. 完全没有训练数据时候，已就绪
+    3. 有训练数据，中间过程全部是进行中
+  */
+  const statesArray = useMemo(() => {
+    const isReady =
+      Object.values(trainingDetail.queuedCounts).every((count) => count === 0) &&
+      Object.values(trainingDetail.trainingCounts).every((count) => count === 0) &&
+      Object.values(trainingDetail.errorCounts).every((count) => count === 0);
+
+    const getTrainingStatus = ({ errorCount }: { errorCount: number }) => {
+      if (isReady) return TrainingStatus.Ready;
+      if (errorCount > 0) {
+        return TrainingStatus.Error;
+      }
+      return TrainingStatus.Running;
+    };
+
+    // 只显示排队和处理中的数量
+    const getStatusText = (mode: TrainingModeEnum) => {
+      if (isReady) return;
+
+      if (trainingDetail.queuedCounts[mode] > 0) {
+        return t('dataset:dataset.Training_Waiting', {
+          count: trainingDetail.queuedCounts[mode]
+        });
+      }
+      if (trainingDetail.trainingCounts[mode] > 0) {
+        return t('dataset:dataset.Training_Count', {
+          count: trainingDetail.trainingCounts[mode]
+        });
+      }
+      return;
+    };
+
+    const states: {
+      label: string;
+      statusText?: string;
+      status: TrainingStatus;
+      errorCount: number;
+    }[] = [
+      // {
+      //   label: TrainingProcess.waiting.label,
+      //   status: TrainingStatus.Queued,
+      //   statusText: t('dataset:dataset.Completed')
+      // },
+      {
+        label: t(TrainingProcess.parsing.label),
+        status: TrainingStatus.Ready,
+        errorCount: 0
+      },
+      ...(isQA
+        ? [
+            {
+              errorCount: trainingDetail.errorCounts.qa,
+              label: t(TrainingProcess.getQA.label),
+              statusText: getStatusText(TrainingModeEnum.qa),
+              status: getTrainingStatus({
+                errorCount: trainingDetail.errorCounts.qa
+              })
+            }
+          ]
+        : []),
+      ...(trainingDetail?.advancedTraining.imageIndex && !isQA
+        ? [
+            {
+              errorCount: trainingDetail.errorCounts.image,
+              label: t(TrainingProcess.imageIndex.label),
+              statusText: getStatusText(TrainingModeEnum.image),
+              status: getTrainingStatus({
+                errorCount: trainingDetail.errorCounts.image
+              })
+            }
+          ]
+        : []),
+      ...(trainingDetail?.advancedTraining.autoIndexes && !isQA
+        ? [
+            {
+              errorCount: trainingDetail.errorCounts.auto,
+              label: t(TrainingProcess.autoIndex.label),
+              statusText: getStatusText(TrainingModeEnum.auto),
+              status: getTrainingStatus({
+                errorCount: trainingDetail.errorCounts.auto
+              })
+            }
+          ]
+        : []),
+      {
+        errorCount: trainingDetail.errorCounts.chunk,
+        label: t(TrainingProcess.vectorizing.label),
+        statusText: getStatusText(TrainingModeEnum.chunk),
+        status: getTrainingStatus({
+          errorCount: trainingDetail.errorCounts.chunk
+        })
+      },
+      {
+        errorCount: 0,
+        label: t('dataset:process.Is_Ready'),
+        status: isReady ? TrainingStatus.Ready : TrainingStatus.NotStart,
+        statusText: isReady
+          ? undefined
+          : t('dataset:training_ready', {
+              count: trainingDetail.trainedCount
+            })
+      }
+    ];
+
+    return states;
+  }, [trainingDetail, t, isQA]);
+
+  return (
+    <Flex flexDirection={'column'} gap={6}>
+      {statesArray.map((item, index) => (
+        <Flex alignItems={'center'} pl={4} key={index}>
+          {/* Status round */}
+          <Box
+            w={'14px'}
+            h={'14px'}
+            borderWidth={'2px'}
+            borderRadius={'50%'}
+            position={'relative'}
+            display={'flex'}
+            alignItems={'center'}
+            justifyContent={'center'}
+            {...((item.status === TrainingStatus.Running ||
+              item.status === TrainingStatus.Error) && {
+              bg: 'primary.600',
+              borderColor: 'primary.600',
+              boxShadow: '0 0 0 4px var(--Royal-Blue-100, #E1EAFF)'
+            })}
+            {...(item.status === TrainingStatus.Ready && {
+              bg: 'primary.600',
+              borderColor: 'primary.600'
+            })}
+            // Line
+            {...(index !== statesArray.length - 1 && {
+              _after: {
+                content: '""',
+                height: '59px',
+                width: '2px',
+                bgColor: 'myGray.250',
+                position: 'absolute',
+                top: '14px',
+                left: '4px'
+              }
+            })}
+          >
+            {item.status === TrainingStatus.Ready && (
+              <MyIcon name="common/check" w={3} color={'white'} />
+            )}
+          </Box>
+          {/* Card */}
+          <Flex
+            alignItems={'center'}
+            w={'full'}
+            bg={
+              item.status === TrainingStatus.Running
+                ? 'primary.50'
+                : item.status === TrainingStatus.Error
+                  ? 'red.50'
+                  : 'myGray.50'
+            }
+            py={2.5}
+            px={3}
+            ml={5}
+            borderRadius={'8px'}
+            flex={1}
+            h={'53px'}
+          >
+            <Box
+              fontSize={'14px'}
+              fontWeight={'medium'}
+              color={item.status === TrainingStatus.NotStart ? 'myGray.400' : 'myGray.900'}
+              mr={2}
+            >
+              {t(item.label as any)}
+            </Box>
+            {item.status === TrainingStatus.Error && (
+              <MyTag
+                showDot
+                type={'borderSolid'}
+                px={1}
+                fontSize={'mini'}
+                borderRadius={'md'}
+                h={5}
+                colorSchema={'red'}
+              >
+                {t('dataset:training.Error', { count: item.errorCount })}
+              </MyTag>
+            )}
+            <Box flex={1} />
+            {!!item.statusText && (
+              <Flex fontSize={'sm'} alignItems={'center'}>
+                {item.statusText}
+              </Flex>
+            )}
+          </Flex>
+        </Flex>
+      ))}
+    </Flex>
+  );
+};
+
+const ErrorView = ({ datasetId, collectionId }: { datasetId: string; collectionId: string }) => {
+  const { t } = useTranslation();
+  const TrainingText = {
+    [TrainingModeEnum.chunk]: t('dataset:process.Vectorizing'),
+    [TrainingModeEnum.qa]: t('dataset:process.Get QA'),
+    [TrainingModeEnum.image]: t('dataset:process.Image_Index'),
+    [TrainingModeEnum.auto]: t('dataset:process.Auto_Index')
+  };
+
+  const [editChunk, setEditChunk] = useState<getTrainingDataDetailResponse>();
+
+  const {
+    data: errorList,
+    ScrollData,
+    isLoading,
+    refreshList
+  } = useScrollPagination(getTrainingError, {
+    pageSize: 15,
+    params: {
+      collectionId
+    },
+    EmptyTip: <EmptyTip />
+  });
+
+  const { runAsync: getData, loading: getDataLoading } = useRequest2(
+    (data: { datasetId: string; collectionId: string; dataId: string }) => {
+      return getTrainingDataDetail(data);
+    },
+    {
+      manual: true,
+      onSuccess: (data) => {
+        setEditChunk(data);
+      }
+    }
+  );
+  const { runAsync: deleteData, loading: deleteLoading } = useRequest2(
+    (data: { datasetId: string; collectionId: string; dataId: string }) => {
+      return deleteTrainingData(data);
+    },
+    {
+      manual: true,
+      onSuccess: () => {
+        refreshList();
+      }
+    }
+  );
+  const { runAsync: updateData, loading: updateLoading } = useRequest2(
+    (data: { datasetId: string; collectionId: string; dataId: string; q?: string; a?: string }) => {
+      return updateTrainingData(data);
+    },
+    {
+      manual: true,
+      onSuccess: () => {
+        refreshList();
+        setEditChunk(undefined);
+      }
+    }
+  );
+
+  if (editChunk) {
+    return (
+      <EditView
+        editChunk={editChunk}
+        onCancel={() => setEditChunk(undefined)}
+        onSave={(data) => {
+          updateData({
+            datasetId,
+            collectionId,
+            dataId: editChunk._id,
+            ...data
+          });
+        }}
+      />
+    );
+  }
+
+  return (
+    <ScrollData
+      h={'400px'}
+      isLoading={isLoading || updateLoading || getDataLoading || deleteLoading}
+    >
+      <TableContainer overflowY={'auto'} fontSize={'12px'}>
+        <Table variant={'simple'}>
+          <Thead>
+            <Tr>
+              <Th pr={0}>{t('dataset:dataset.Chunk_Number')}</Th>
+              <Th pr={0}>{t('dataset:dataset.Training_Status')}</Th>
+              <Th>{t('dataset:dataset.Error_Message')}</Th>
+              <Th>{t('dataset:dataset.Operation')}</Th>
+            </Tr>
+          </Thead>
+          <Tbody>
+            {errorList.map((item, index) => (
+              <Tr key={index}>
+                <Td>{item.chunkIndex + 1}</Td>
+                <Td>{TrainingText[item.mode]}</Td>
+                <Td maxW={50}>
+                  <MyTooltip label={item.errorMsg}>{item.errorMsg}</MyTooltip>
+                </Td>
+                <Td>
+                  <Flex alignItems={'center'}>
+                    <Button
+                      variant={'ghost'}
+                      size={'sm'}
+                      color={'myGray.600'}
+                      leftIcon={<MyIcon name={'common/confirm/restoreTip'} w={4} />}
+                      fontSize={'mini'}
+                      onClick={() => updateData({ datasetId, collectionId, dataId: item._id })}
+                    >
+                      {t('dataset:dataset.ReTrain')}
+                    </Button>
+                    <Box w={'1px'} height={'16px'} bg={'myGray.200'} />
+                    <Button
+                      variant={'ghost'}
+                      size={'sm'}
+                      color={'myGray.600'}
+                      leftIcon={<MyIcon name={'edit'} w={4} />}
+                      fontSize={'mini'}
+                      onClick={() => getData({ datasetId, collectionId, dataId: item._id })}
+                    >
+                      {t('dataset:dataset.Edit_Chunk')}
+                    </Button>
+                    <Box w={'1px'} height={'16px'} bg={'myGray.200'} />
+                    <Button
+                      variant={'ghost'}
+                      size={'sm'}
+                      color={'myGray.600'}
+                      leftIcon={<MyIcon name={'delete'} w={4} />}
+                      fontSize={'mini'}
+                      onClick={() => {
+                        deleteData({ datasetId, collectionId, dataId: item._id });
+                      }}
+                    >
+                      {t('dataset:dataset.Delete_Chunk')}
+                    </Button>
+                  </Flex>
+                </Td>
+              </Tr>
+            ))}
+          </Tbody>
+        </Table>
+      </TableContainer>
+    </ScrollData>
+  );
+};
+
+const EditView = ({
+  editChunk,
+  onCancel,
+  onSave
+}: {
+  editChunk: getTrainingDataDetailResponse;
+  onCancel: () => void;
+  onSave: (data: { q: string; a?: string }) => void;
+}) => {
+  const { t } = useTranslation();
+  const { register, handleSubmit } = useForm({
+    defaultValues: {
+      q: editChunk?.q || '',
+      a: editChunk?.a || ''
+    }
+  });
+
+  return (
+    <Flex flexDirection={'column'} gap={4}>
+      {editChunk?.a && <Box>q</Box>}
+      <MyTextarea {...register('q')} minH={editChunk?.a ? 200 : 400} />
+      {editChunk?.a && (
+        <>
+          <Box>a</Box>
+          <MyTextarea {...register('a')} minH={200} />
+        </>
+      )}
+      <Flex justifyContent={'flex-end'} gap={4}>
+        <Button variant={'outline'} onClick={onCancel}>
+          {t('common:common.Cancel')}
+        </Button>
+        <Button variant={'primary'} onClick={handleSubmit(onSave)}>
+          {t('dataset:dataset.ReTrain')}
+        </Button>
+      </Flex>
+    </Flex>
+  );
+};
+
+const TrainingStates = ({
+  datasetId,
+  collectionId,
+  defaultTab = 'states',
+  onClose
+}: {
+  datasetId: string;
+  collectionId: string;
+  defaultTab?: 'states' | 'errors';
+  onClose: () => void;
+}) => {
+  const { t } = useTranslation();
+  const [tab, setTab] = useState<typeof defaultTab>(defaultTab);
+
+  const { data: trainingDetail, loading } = useRequest2(
+    () => getDatasetCollectionTrainingDetail(collectionId),
+    {
+      pollingInterval: 5000,
+      pollingWhenHidden: false,
+      manual: false
+    }
+  );
+
+  const errorCounts = (Object.values(trainingDetail?.errorCounts || {}) as number[]).reduce(
+    (acc, count) => acc + count,
+    0
+  );
+
+  return (
+    <MyModal
+      isOpen
+      onClose={onClose}
+      iconSrc="common/running"
+      title={t('dataset:dataset.Training Process')}
+      minW={['90vw', '712px']}
+      isLoading={!trainingDetail && loading && tab === 'states'}
+    >
+      <ModalBody px={9} minH={['90vh', '500px']}>
+        <FillRowTabs
+          py={1}
+          mb={6}
+          value={tab}
+          onChange={(e) => setTab(e as 'states' | 'errors')}
+          list={[
+            { label: t('dataset:dataset.Training Process'), value: 'states' },
+            {
+              label: t('dataset:dataset.Training_Errors', {
+                count: errorCounts
+              }),
+              value: 'errors'
+            }
+          ]}
+        />
+        {tab === 'states' && trainingDetail && <ProgressView trainingDetail={trainingDetail} />}
+        {tab === 'errors' && <ErrorView datasetId={datasetId} collectionId={collectionId} />}
+      </ModalBody>
+    </MyModal>
+  );
+};
+
+export default TrainingStates;

projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx

@@ -1,110 +1,215 @@
-import React from 'react';
 import MyModal from '@fastgpt/web/components/common/MyModal';
 import { useTranslation } from 'next-i18next';
-import { Box, Button, Input, Link, ModalBody, ModalFooter } from '@chakra-ui/react';
 import { strIsLink } from '@fastgpt/global/common/string/tools';
 import { useToast } from '@fastgpt/web/hooks/useToast';
 import { useForm } from 'react-hook-form';
 import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
 import { getDocPath } from '@/web/common/system/doc';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
+import { useMyStep } from '@fastgpt/web/hooks/useStep';
+import MyDivider from '@fastgpt/web/components/common/MyDivider';
+import React, { useRef } from 'react';
+import {
+  Box,
+  Link,
+  Input,
+  Button,
+  ModalBody,
+  ModalFooter,
+  Textarea,
+  Stack
+} from '@chakra-ui/react';
+import {
+  DataChunkSplitModeEnum,
+  DatasetCollectionDataProcessModeEnum
+} from '@fastgpt/global/core/dataset/constants';
+import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
+import { useContextSelector } from 'use-context-selector';
+import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import CollectionChunkForm, {
+  collectionChunkForm2StoreChunkData,
+  type CollectionChunkFormType
+} from '../Form/CollectionChunkForm';
+import { getLLMDefaultChunkSize } from '@fastgpt/global/core/dataset/training/utils';
+import { ChunkSettingsType } from '@fastgpt/global/core/dataset/type';
 
-type FormType = {
-  url?: string | undefined;
-  selector?: string | undefined;
+export type WebsiteConfigFormType = {
+  websiteConfig: {
+    url: string;
+    selector: string;
+  };
+  chunkSettings: ChunkSettingsType;
 };
 
 const WebsiteConfigModal = ({
   onClose,
-  onSuccess,
-  defaultValue = {
-    url: '',
-    selector: ''
-  }
+  onSuccess
 }: {
   onClose: () => void;
-  onSuccess: (data: FormType) => void;
-  defaultValue?: FormType;
+  onSuccess: (data: WebsiteConfigFormType) => void;
 }) => {
   const { t } = useTranslation();
   const { feConfigs } = useSystemStore();
   const { toast } = useToast();
-  const { register, handleSubmit } = useForm({
-    defaultValues: defaultValue
+  const steps = [
+    {
+      title: t('dataset:website_info')
+    },
+    {
+      title: t('dataset:params_config')
+    }
+  ];
+
+  const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
+  const websiteConfig = datasetDetail.websiteConfig;
+  const chunkSettings = datasetDetail.chunkSettings;
+
+  const {
+    register: websiteInfoForm,
+    handleSubmit: websiteInfoHandleSubmit,
+    getValues: websiteInfoGetValues
+  } = useForm({
+    defaultValues: {
+      url: websiteConfig?.url || '',
+      selector: websiteConfig?.selector || ''
+    }
   });
-  const isEdit = !!defaultValue.url;
-  const confirmTip = isEdit
-    ? t('common:core.dataset.website.Confirm Update Tips')
-    : t('common:core.dataset.website.Confirm Create Tips');
+
+  const isEdit = !!websiteConfig?.url;
 
   const { ConfirmModal, openConfirm } = useConfirm({
     type: 'common'
   });
 
+  const { activeStep, goToPrevious, goToNext, MyStep } = useMyStep({
+    defaultStep: 0,
+    steps
+  });
+
+  const form = useForm<CollectionChunkFormType>({
+    defaultValues: {
+      trainingType: chunkSettings?.trainingType || DatasetCollectionDataProcessModeEnum.chunk,
+      imageIndex: chunkSettings?.imageIndex || false,
+      autoIndexes: chunkSettings?.autoIndexes || false,
+
+      chunkSettingMode: chunkSettings?.chunkSettingMode || ChunkSettingModeEnum.auto,
+      chunkSplitMode: chunkSettings?.chunkSplitMode || DataChunkSplitModeEnum.size,
+      embeddingChunkSize: chunkSettings?.chunkSize || 2000,
+      qaChunkSize: chunkSettings?.chunkSize || getLLMDefaultChunkSize(datasetDetail.agentModel),
+      indexSize: chunkSettings?.indexSize || datasetDetail.vectorModel?.defaultToken || 512,
+
+      chunkSplitter: chunkSettings?.chunkSplitter || '',
+      qaPrompt: chunkSettings?.qaPrompt || Prompt_AgentQA.description
+    }
+  });
+
   return (
     <MyModal
       isOpen
       iconSrc="core/dataset/websiteDataset"
       title={t('common:core.dataset.website.Config')}
       onClose={onClose}
-      maxW={'500px'}
+      w={'550px'}
     >
-      <ModalBody>
-        <Box fontSize={'sm'} color={'myGray.600'}>
-          {t('common:core.dataset.website.Config Description')}
-          {feConfigs?.docUrl && (
-            <Link
-              href={getDocPath('/docs/guide/knowledge_base/websync/')}
-              target="_blank"
-              textDecoration={'underline'}
-              fontWeight={'bold'}
+      <ModalBody w={'full'}>
+        <Stack w={'75%'} marginX={'auto'}>
+          <MyStep />
+        </Stack>
+        <MyDivider />
+        {activeStep == 0 && (
+          <>
+            <Box
+              fontSize={'xs'}
+              color={'myGray.900'}
+              bgColor={'blue.50'}
+              padding={'4'}
+              borderRadius={'8px'}
             >
-              {t('common:common.course.Read Course')}
-            </Link>
-          )}
-        </Box>
-        <Box mt={2}>
-          <Box>{t('common:core.dataset.website.Base Url')}</Box>
-          <Input
-            placeholder={t('common:core.dataset.collection.Website Link')}
-            {...register('url', {
-              required: true
-            })}
-          />
-        </Box>
-        <Box mt={3}>
-          <Box>
-            {t('common:core.dataset.website.Selector')}({t('common:common.choosable')})
-          </Box>
-          <Input {...register('selector')} placeholder="body .content #document" />
-        </Box>
+              {t('common:core.dataset.website.Config Description')}
+              {feConfigs?.docUrl && (
+                <Link
+                  href={getDocPath('/docs/guide/knowledge_base/websync/')}
+                  target="_blank"
+                  textDecoration={'underline'}
+                  color={'blue.700'}
+                >
+                  {t('common:common.course.Read Course')}
+                </Link>
+              )}
+            </Box>
+            <Box mt={2}>
+              <Box>{t('common:core.dataset.website.Base Url')}</Box>
+              <Input
+                placeholder={t('common:core.dataset.collection.Website Link')}
+                {...websiteInfoForm('url', {
+                  required: true
+                })}
+              />
+            </Box>
+            <Box mt={3}>
+              <Box>
+                {t('common:core.dataset.website.Selector')}({t('common:common.choosable')})
+              </Box>
+              <Input {...websiteInfoForm('selector')} placeholder="body .content #document" />
+            </Box>
+          </>
+        )}
+        {activeStep == 1 && <CollectionChunkForm form={form} />}
       </ModalBody>
       <ModalFooter>
-        <Button variant={'whiteBase'} onClick={onClose}>
-          {t('common:common.Close')}
-        </Button>
-        <Button
-          ml={2}
-          onClick={handleSubmit((data) => {
-            if (!data.url) return;
-            // check is link
-            if (!strIsLink(data.url)) {
-              return toast({
-                status: 'warning',
-                title: t('common:common.link.UnValid')
-              });
-            }
-            openConfirm(
-              () => {
-                onSuccess(data);
-              },
-              undefined,
-              confirmTip
-            )();
-          })}
-        >
-          {t('common:core.dataset.website.Start Sync')}
-        </Button>
+        {activeStep == 0 && (
+          <>
+            <Button variant={'whiteBase'} onClick={onClose}>
+              {t('common:common.Close')}
+            </Button>
+            <Button
+              ml={2}
+              onClick={websiteInfoHandleSubmit((data) => {
+                if (!data.url) return;
+                // check is link
+                if (!strIsLink(data.url)) {
+                  return toast({
+                    status: 'warning',
+                    title: t('common:common.link.UnValid')
+                  });
+                }
+                goToNext();
+              })}
+            >
+              {t('common:common.Next Step')}
+            </Button>
+          </>
+        )}
+        {activeStep == 1 && (
+          <>
+            <Button variant={'whiteBase'} onClick={goToPrevious}>
+              {t('common:common.Last Step')}
+            </Button>
+            <Button
+              ml={2}
+              onClick={form.handleSubmit((data) => {
+                openConfirm(
+                  () =>
+                    onSuccess({
+                      websiteConfig: websiteInfoGetValues(),
+                      chunkSettings: collectionChunkForm2StoreChunkData({
+                        ...data,
+                        agentModel: datasetDetail.agentModel,
+                        vectorModel: datasetDetail.vectorModel
+                      })
+                    }),
+                  undefined,
+                  isEdit
+                    ? t('common:core.dataset.website.Confirm Update Tips')
+                    : t('common:core.dataset.website.Confirm Create Tips')
+                )();
+              })}
+            >
+              {t('common:core.dataset.website.Start Sync')}
+            </Button>
+          </>
+        )}
       </ModalFooter>
       <ConfirmModal />
     </MyModal>
@@ -112,3 +217,42 @@ const WebsiteConfigModal = ({
 };
 
 export default WebsiteConfigModal;
+
+const PromptTextarea = ({
+  defaultValue,
+  onChange,
+  onClose
+}: {
+  defaultValue: string;
+  onChange: (e: string) => void;
+  onClose: () => void;
+}) => {
+  const ref = useRef<HTMLTextAreaElement>(null);
+  const { t } = useTranslation();
+
+  return (
+    <MyModal
+      isOpen
+      title={t('common:core.dataset.import.Custom prompt')}
+      iconSrc="modal/edit"
+      w={'600px'}
+      onClose={onClose}
+    >
+      <ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
+        <Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
+        <Box>{Prompt_AgentQA.fixedText}</Box>
+      </ModalBody>
+      <ModalFooter>
+        <Button
+          onClick={() => {
+            const val = ref.current?.value || Prompt_AgentQA.description;
+            onChange(val);
+            onClose();
+          }}
+        >
+          {t('common:common.Confirm')}
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};

projects/app/src/pageComponents/dataset/detail/CollectionCard/index.tsx

@@ -51,6 +51,7 @@ import {
 import { useFolderDrag } from '@/components/common/folder/useFolderDrag';
 import TagsPopOver from './TagsPopOver';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
+import TrainingStates from './TrainingStates';
 
 const Header = dynamic(() => import('./Header'));
 const EmptyCollectionTip = dynamic(() => import('./EmptyCollectionTip'));
@@ -63,26 +64,25 @@ const CollectionCard = () => {
   const { datasetDetail, loadDatasetDetail } = useContextSelector(DatasetPageContext, (v) => v);
   const { feConfigs } = useSystemStore();
 
-  const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
-    content: t('common:dataset.Confirm to delete the file'),
-    type: 'delete'
-  });
-
-  const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
-    title: t('common:Rename')
-  });
-
-  const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>();
+  const [trainingStatesCollection, setTrainingStatesCollection] = useState<{
+    collectionId: string;
+  }>();
 
   const { collections, Pagination, total, getData, isGetting, pageNum, pageSize } =
     useContextSelector(CollectionPageContext, (v) => v);
 
-  // Ad file status icon
+  // Add file status icon
   const formatCollections = useMemo(
     () =>
       collections.map((collection) => {
         const icon = getCollectionIcon(collection.type, collection.name);
         const status = (() => {
+          if (collection.hasError) {
+            return {
+              statusText: t('common:core.dataset.collection.status.error'),
+              colorSchema: 'red'
+            };
+          }
           if (collection.trainingAmount > 0) {
             return {
               statusText: t('common:dataset.collections.Collection Embedding', {
@@ -106,6 +106,11 @@ const CollectionCard = () => {
     [collections, t]
   );
 
+  const [moveCollectionData, setMoveCollectionData] = useState<{ collectionId: string }>();
+
+  const { onOpenModal: onOpenEditTitleModal, EditModal: EditTitleModal } = useEditTitle({
+    title: t('common:Rename')
+  });
   const { runAsync: onUpdateCollection, loading: isUpdating } = useRequest2(
     putDatasetCollectionById,
     {
@@ -115,7 +120,12 @@ const CollectionCard = () => {
       successToast: t('common:common.Update Success')
     }
   );
-  const { runAsync: onDelCollection, loading: isDeleting } = useRequest2(
+
+  const { openConfirm: openDeleteConfirm, ConfirmModal: ConfirmDeleteModal } = useConfirm({
+    content: t('common:dataset.Confirm to delete the file'),
+    type: 'delete'
+  });
+  const { runAsync: onDelCollection } = useRequest2(
     (collectionId: string) => {
       return delDatasetCollectionById({
         id: collectionId
@@ -153,14 +163,14 @@ const CollectionCard = () => {
     ['refreshCollection'],
     () => {
       getData(pageNum);
-      if (datasetDetail.status === DatasetStatusEnum.syncing) {
+      if (datasetDetail.status !== DatasetStatusEnum.active) {
         loadDatasetDetail(datasetDetail._id);
       }
       return null;
     },
     {
       refetchInterval: 6000,
-      enabled: hasTrainingData || datasetDetail.status === DatasetStatusEnum.syncing
+      enabled: hasTrainingData || datasetDetail.status !== DatasetStatusEnum.active
     }
   );
 
@@ -180,13 +190,13 @@ const CollectionCard = () => {
   });
 
   const isLoading =
-    isUpdating || isDeleting || isSyncing || (isGetting && collections.length === 0) || isDropping;
+    isUpdating || isSyncing || (isGetting && collections.length === 0) || isDropping;
 
   return (
     <MyBox isLoading={isLoading} h={'100%'} py={[2, 4]}>
       <Flex ref={BoxRef} flexDirection={'column'} py={[1, 0]} h={'100%'} px={[2, 6]}>
         {/* header */}
-        <Header />
+        <Header hasTrainingData={hasTrainingData} />
 
         {/* collection table */}
         <TableContainer mt={3} overflowY={'auto'} fontSize={'sm'}>
@@ -269,9 +279,22 @@ const CollectionCard = () => {
                     <Box>{formatTime2YMDHM(collection.updateTime)}</Box>
                   </Td>
                   <Td py={2}>
-                    <MyTag showDot colorSchema={collection.colorSchema as any} type={'borderFill'}>
-                      {t(collection.statusText as any)}
-                    </MyTag>
+                    <MyTooltip label={t('common:Click_to_expand')}>
+                      <MyTag
+                        showDot
+                        colorSchema={collection.colorSchema as any}
+                        type={'fill'}
+                        onClick={(e) => {
+                          e.stopPropagation();
+                          setTrainingStatesCollection({ collectionId: collection._id });
+                        }}
+                      >
+                        <Flex fontWeight={'medium'} alignItems={'center'} gap={1}>
+                          {t(collection.statusText as any)}
+                          <MyIcon name={'common/maximize'} w={'11px'} />
+                        </Flex>
+                      </MyTag>
+                    </MyTooltip>
                   </Td>
                   <Td py={2} onClick={(e) => e.stopPropagation()}>
                     <Switch
@@ -383,9 +406,7 @@ const CollectionCard = () => {
                                 type: 'danger',
                                 onClick: () =>
                                   openDeleteConfirm(
-                                    () => {
-                                      onDelCollection(collection._id);
-                                    },
+                                    () => onDelCollection(collection._id),
                                     undefined,
                                     collection.type === DatasetCollectionTypeEnum.folder
                                       ? t('common:dataset.collections.Confirm to delete the folder')
@@ -414,6 +435,14 @@ const CollectionCard = () => {
         <ConfirmSyncModal />
         <EditTitleModal />
 
+        {!!trainingStatesCollection && (
+          <TrainingStates
+            datasetId={datasetDetail._id}
+            collectionId={trainingStatesCollection.collectionId}
+            onClose={() => setTrainingStatesCollection(undefined)}
+          />
+        )}
+
         {!!moveCollectionData && (
           <SelectCollections
             datasetId={datasetDetail._id}

projects/app/src/pageComponents/dataset/detail/DataCard.tsx

@@ -30,6 +30,7 @@ import { useScrollPagination } from '@fastgpt/web/hooks/useScrollPagination';
 import { TabEnum } from './NavBar';
 import { ImportDataSourceEnum } from '@fastgpt/global/core/dataset/constants';
 import { useRequest2 } from '@fastgpt/web/hooks/useRequest';
+import TrainingStates from './CollectionCard/TrainingStates';
 
 const DataCard = () => {
   const theme = useTheme();
@@ -44,6 +45,7 @@ const DataCard = () => {
 
   const { t } = useTranslation();
   const [searchText, setSearchText] = useState('');
+  const [errorModalId, setErrorModalId] = useState('');
   const { toast } = useToast();
 
   const scrollParams = useMemo(
@@ -174,7 +176,7 @@ const DataCard = () => {
           <MyDivider my={'17px'} w={'100%'} />
         </Box>
         <Flex alignItems={'center'} px={6} pb={4}>
-          <Flex align={'center'} color={'myGray.500'}>
+          <Flex alignItems={'center'} color={'myGray.500'}>
             <MyIcon name="common/list" mr={2} w={'18px'} />
             <Box as={'span'} fontSize={['sm', '14px']} fontWeight={'500'}>
               {t('dataset:data_amount', {
@@ -182,6 +184,25 @@ const DataCard = () => {
                 indexAmount: collection?.indexAmount ?? '-'
               })}
             </Box>
+            {!!collection?.errorCount && (
+              <MyTag
+                colorSchema={'red'}
+                type={'fill'}
+                cursor={'pointer'}
+                rounded={'full'}
+                ml={2}
+                onClick={() => {
+                  setErrorModalId(collection._id);
+                }}
+              >
+                <Flex fontWeight={'medium'} alignItems={'center'} gap={1}>
+                  {t('dataset:data_error_amount', {
+                    errorAmount: collection?.errorCount
+                  })}
+                  <MyIcon name={'common/maximize'} w={'11px'} />
+                </Flex>
+              </MyTag>
+            )}
           </Flex>
           <Box flex={1} mr={1} />
           <MyInput
@@ -354,6 +375,14 @@ const DataCard = () => {
           }}
         />
       )}
+      {errorModalId && (
+        <TrainingStates
+          datasetId={datasetId}
+          defaultTab={'errors'}
+          collectionId={errorModalId}
+          onClose={() => setErrorModalId('')}
+        />
+      )}
       <ConfirmModal />
     </MyBox>
   );

projects/app/src/pageComponents/dataset/detail/Form/CollectionChunkForm.tsx

@@ -0,0 +1,524 @@
+import MyModal from '@fastgpt/web/components/common/MyModal';
+import { useTranslation } from 'next-i18next';
+import { UseFormReturn } from 'react-hook-form';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import React, { useEffect, useMemo, useRef, useState } from 'react';
+import {
+  Box,
+  Flex,
+  Input,
+  Button,
+  ModalBody,
+  ModalFooter,
+  Textarea,
+  useDisclosure,
+  Checkbox,
+  HStack
+} from '@chakra-ui/react';
+import MyIcon from '@fastgpt/web/components/common/Icon';
+import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
+import {
+  DataChunkSplitModeEnum,
+  DatasetCollectionDataProcessModeEnum,
+  DatasetCollectionDataProcessModeMap
+} from '@fastgpt/global/core/dataset/constants';
+import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
+import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
+import { useContextSelector } from 'use-context-selector';
+import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
+import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
+import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
+import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import MySelect from '@fastgpt/web/components/common/MySelect';
+import {
+  chunkAutoChunkSize,
+  getAutoIndexSize,
+  getIndexSizeSelectList,
+  getLLMDefaultChunkSize,
+  getLLMMaxChunkSize,
+  getMaxChunkSize,
+  getMaxIndexSize,
+  minChunkSize
+} from '@fastgpt/global/core/dataset/training/utils';
+import RadioGroup from '@fastgpt/web/components/common/Radio/RadioGroup';
+import { ChunkSettingsType } from '@fastgpt/global/core/dataset/type';
+import type { LLMModelItemType, EmbeddingModelItemType } from '@fastgpt/global/core/ai/model.d';
+
+const PromptTextarea = ({
+  defaultValue = '',
+  onChange,
+  onClose
+}: {
+  defaultValue?: string;
+  onChange: (e: string) => void;
+  onClose: () => void;
+}) => {
+  const ref = useRef<HTMLTextAreaElement>(null);
+  const { t } = useTranslation();
+
+  return (
+    <MyModal
+      isOpen
+      title={t('common:core.dataset.import.Custom prompt')}
+      iconSrc="modal/edit"
+      w={'600px'}
+      onClose={onClose}
+    >
+      <ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
+        <Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
+        <Box>{Prompt_AgentQA.fixedText}</Box>
+      </ModalBody>
+      <ModalFooter>
+        <Button
+          onClick={() => {
+            const val = ref.current?.value || Prompt_AgentQA.description;
+            onChange(val);
+            onClose();
+          }}
+        >
+          {t('common:common.Confirm')}
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};
+
+export type CollectionChunkFormType = {
+  trainingType: DatasetCollectionDataProcessModeEnum;
+  imageIndex: boolean;
+  autoIndexes: boolean;
+
+  chunkSettingMode: ChunkSettingModeEnum;
+
+  chunkSplitMode: DataChunkSplitModeEnum;
+  embeddingChunkSize: number;
+  qaChunkSize: number;
+  chunkSplitter?: string;
+  indexSize: number;
+
+  qaPrompt?: string;
+};
+const CollectionChunkForm = ({ form }: { form: UseFormReturn<CollectionChunkFormType> }) => {
+  const { t } = useTranslation();
+  const { feConfigs } = useSystemStore();
+
+  const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
+
+  const vectorModel = datasetDetail.vectorModel;
+  const agentModel = datasetDetail.agentModel;
+
+  const { setValue, register, watch, getValues } = form;
+
+  const trainingType = watch('trainingType');
+  const chunkSettingMode = watch('chunkSettingMode');
+  const chunkSplitMode = watch('chunkSplitMode');
+  const autoIndexes = watch('autoIndexes');
+  const indexSize = watch('indexSize');
+
+  const trainingModeList = useMemo(() => {
+    const list = Object.entries(DatasetCollectionDataProcessModeMap);
+    return list
+      .filter(([key]) => key !== DatasetCollectionDataProcessModeEnum.auto)
+      .map(([key, value]) => ({
+        title: t(value.label as any),
+        value: key as DatasetCollectionDataProcessModeEnum,
+        tooltip: t(value.tooltip as any)
+      }));
+  }, [t]);
+  const {
+    chunkSizeField,
+    maxChunkSize,
+    minChunkSize: minChunkSizeValue,
+    maxIndexSize
+  } = useMemo(() => {
+    if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
+      return {
+        chunkSizeField: 'qaChunkSize',
+        maxChunkSize: getLLMMaxChunkSize(agentModel),
+        minChunkSize: 1000,
+        maxIndexSize: 1000
+      };
+    } else if (autoIndexes) {
+      return {
+        chunkSizeField: 'embeddingChunkSize',
+        maxChunkSize: getMaxChunkSize(agentModel),
+        minChunkSize: minChunkSize,
+        maxIndexSize: getMaxIndexSize(vectorModel)
+      };
+    } else {
+      return {
+        chunkSizeField: 'embeddingChunkSize',
+        maxChunkSize: getMaxChunkSize(agentModel),
+        minChunkSize: minChunkSize,
+        maxIndexSize: getMaxIndexSize(vectorModel)
+      };
+    }
+  }, [trainingType, autoIndexes, agentModel, vectorModel]);
+
+  // Custom split list
+  const customSplitList = [
+    { label: t('dataset:split_sign_null'), value: '' },
+    { label: t('dataset:split_sign_break'), value: '\\n' },
+    { label: t('dataset:split_sign_break2'), value: '\\n\\n' },
+    { label: t('dataset:split_sign_period'), value: '.|。' },
+    { label: t('dataset:split_sign_exclamatiob'), value: '!|！' },
+    { label: t('dataset:split_sign_question'), value: '?|？' },
+    { label: t('dataset:split_sign_semicolon'), value: ';|；' },
+    { label: '=====', value: '=====' },
+    { label: t('dataset:split_sign_custom'), value: 'Other' }
+  ];
+  const [customListSelectValue, setCustomListSelectValue] = useState(getValues('chunkSplitter'));
+  useEffect(() => {
+    if (customListSelectValue === 'Other') {
+      setValue('chunkSplitter', '');
+    } else {
+      setValue('chunkSplitter', customListSelectValue);
+    }
+  }, [customListSelectValue, setValue]);
+
+  // Index size
+  const indexSizeSeletorList = useMemo(() => getIndexSizeSelectList(maxIndexSize), [maxIndexSize]);
+
+  // QA
+  const qaPrompt = watch('qaPrompt');
+  const {
+    isOpen: isOpenCustomPrompt,
+    onOpen: onOpenCustomPrompt,
+    onClose: onCloseCustomPrompt
+  } = useDisclosure();
+
+  const showQAPromptInput = trainingType === DatasetCollectionDataProcessModeEnum.qa;
+
+  // Adapt 4.9.0- auto training
+  useEffect(() => {
+    if (trainingType === DatasetCollectionDataProcessModeEnum.auto) {
+      setValue('autoIndexes', true);
+      setValue('trainingType', DatasetCollectionDataProcessModeEnum.chunk);
+    }
+  }, [trainingType, setValue]);
+
+  return (
+    <>
+      <Box>
+        <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
+          {t('dataset:training_mode')}
+        </Box>
+        <LeftRadio<DatasetCollectionDataProcessModeEnum>
+          list={trainingModeList}
+          px={3}
+          py={2.5}
+          value={trainingType}
+          onChange={(e) => {
+            setValue('trainingType', e);
+          }}
+          defaultBg="white"
+          activeBg="white"
+          gridTemplateColumns={'repeat(2, 1fr)'}
+        />
+      </Box>
+      {trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
+        <Box mt={6}>
+          <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
+            {t('dataset:enhanced_indexes')}
+          </Box>
+          <HStack gap={[3, 7]}>
+            <HStack flex={'1'} spacing={1}>
+              <MyTooltip label={!feConfigs?.isPlus ? t('common:commercial_function_tip') : ''}>
+                <Checkbox isDisabled={!feConfigs?.isPlus} {...register('autoIndexes')}>
+                  <FormLabel>{t('dataset:auto_indexes')}</FormLabel>
+                </Checkbox>
+              </MyTooltip>
+              <QuestionTip label={t('dataset:auto_indexes_tips')} />
+            </HStack>
+            <HStack flex={'1'} spacing={1}>
+              <MyTooltip
+                label={
+                  !feConfigs?.isPlus
+                    ? t('common:commercial_function_tip')
+                    : !datasetDetail?.vlmModel
+                      ? t('common:error_vlm_not_config')
+                      : ''
+                }
+              >
+                <Checkbox
+                  isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
+                  {...register('imageIndex')}
+                >
+                  <FormLabel>{t('dataset:image_auto_parse')}</FormLabel>
+                </Checkbox>
+              </MyTooltip>
+              <QuestionTip label={t('dataset:image_auto_parse_tips')} />
+            </HStack>
+          </HStack>
+        </Box>
+      )}
+      <Box mt={6}>
+        <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
+          {t('dataset:params_setting')}
+        </Box>
+        <LeftRadio<ChunkSettingModeEnum>
+          list={[
+            {
+              title: t('dataset:default_params'),
+              desc: t('dataset:default_params_desc'),
+              value: ChunkSettingModeEnum.auto
+            },
+            {
+              title: t('dataset:custom_data_process_params'),
+              desc: t('dataset:custom_data_process_params_desc'),
+              value: ChunkSettingModeEnum.custom,
+              children: chunkSettingMode === ChunkSettingModeEnum.custom && (
+                <Box mt={5}>
+                  <Box>
+                    <RadioGroup<DataChunkSplitModeEnum>
+                      list={[
+                        {
+                          title: t('dataset:split_chunk_size'),
+                          value: DataChunkSplitModeEnum.size
+                        },
+                        {
+                          title: t('dataset:split_chunk_char'),
+                          value: DataChunkSplitModeEnum.char,
+                          tooltip: t('dataset:custom_split_sign_tip')
+                        }
+                      ]}
+                      value={chunkSplitMode}
+                      onChange={(e) => {
+                        setValue('chunkSplitMode', e);
+                      }}
+                    />
+
+                    {chunkSplitMode === DataChunkSplitModeEnum.size && (
+                      <Box
+                        mt={1.5}
+                        css={{
+                          '& > span': {
+                            display: 'block'
+                          }
+                        }}
+                      >
+                        <MyTooltip
+                          label={t('common:core.dataset.import.Chunk Range', {
+                            min: minChunkSizeValue,
+                            max: maxChunkSize
+                          })}
+                        >
+                          <MyNumberInput
+                            register={register}
+                            name={chunkSizeField}
+                            min={minChunkSizeValue}
+                            max={maxChunkSize}
+                            size={'sm'}
+                            step={100}
+                          />
+                        </MyTooltip>
+                      </Box>
+                    )}
+
+                    {chunkSplitMode === DataChunkSplitModeEnum.char && (
+                      <HStack mt={1.5}>
+                        <Box flex={'1 0 0'}>
+                          <MySelect<string>
+                            list={customSplitList}
+                            size={'sm'}
+                            bg={'myGray.50'}
+                            value={customListSelectValue}
+                            h={'32px'}
+                            onChange={(val) => {
+                              setCustomListSelectValue(val);
+                            }}
+                          />
+                        </Box>
+                        {customListSelectValue === 'Other' && (
+                          <Input
+                            flex={'1 0 0'}
+                            h={'32px'}
+                            size={'sm'}
+                            bg={'myGray.50'}
+                            placeholder="\n;======;==SPLIT=="
+                            {...register('chunkSplitter')}
+                          />
+                        )}
+                      </HStack>
+                    )}
+                  </Box>
+
+                  {trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
+                    <Box>
+                      <Flex alignItems={'center'} mt={3}>
+                        <Box>{t('dataset:index_size')}</Box>
+                        <QuestionTip label={t('dataset:index_size_tips')} />
+                      </Flex>
+                      <Box mt={1}>
+                        <MySelect<number>
+                          bg={'myGray.50'}
+                          list={indexSizeSeletorList}
+                          value={indexSize}
+                          onChange={(val) => {
+                            setValue('indexSize', val);
+                          }}
+                        />
+                      </Box>
+                    </Box>
+                  )}
+
+                  {showQAPromptInput && (
+                    <Box mt={3}>
+                      <Box>{t('common:core.dataset.collection.QA Prompt')}</Box>
+                      <Box
+                        position={'relative'}
+                        py={2}
+                        px={3}
+                        bg={'myGray.50'}
+                        fontSize={'xs'}
+                        whiteSpace={'pre-wrap'}
+                        border={'1px'}
+                        borderColor={'borderColor.base'}
+                        borderRadius={'md'}
+                        maxH={'140px'}
+                        overflow={'auto'}
+                        _hover={{
+                          '& .mask': {
+                            display: 'block'
+                          }
+                        }}
+                      >
+                        {qaPrompt}
+
+                        <Box
+                          display={'none'}
+                          className="mask"
+                          position={'absolute'}
+                          top={0}
+                          right={0}
+                          bottom={0}
+                          left={0}
+                          background={
+                            'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
+                          }
+                        >
+                          <Button
+                            size="xs"
+                            variant={'whiteBase'}
+                            leftIcon={<MyIcon name={'edit'} w={'13px'} />}
+                            color={'black'}
+                            position={'absolute'}
+                            right={2}
+                            bottom={2}
+                            onClick={onOpenCustomPrompt}
+                          >
+                            {t('common:core.dataset.import.Custom prompt')}
+                          </Button>
+                        </Box>
+                      </Box>
+                    </Box>
+                  )}
+                </Box>
+              )
+            }
+          ]}
+          gridGap={3}
+          px={3}
+          py={3}
+          defaultBg="white"
+          activeBg="white"
+          value={chunkSettingMode}
+          w={'100%'}
+          onChange={(e) => {
+            setValue('chunkSettingMode', e);
+          }}
+        />
+      </Box>
+      {isOpenCustomPrompt && (
+        <PromptTextarea
+          defaultValue={qaPrompt}
+          onChange={(e) => {
+            setValue('qaPrompt', e);
+          }}
+          onClose={onCloseCustomPrompt}
+        />
+      )}
+    </>
+  );
+};
+
+export default CollectionChunkForm;
+
+export const collectionChunkForm2StoreChunkData = ({
+  trainingType,
+  imageIndex,
+  autoIndexes,
+  chunkSettingMode,
+  chunkSplitMode,
+  embeddingChunkSize,
+  qaChunkSize,
+  chunkSplitter,
+  indexSize,
+  qaPrompt,
+
+  agentModel,
+  vectorModel
+}: CollectionChunkFormType & {
+  agentModel: LLMModelItemType;
+  vectorModel: EmbeddingModelItemType;
+}): ChunkSettingsType => {
+  const trainingModeSize: {
+    autoChunkSize: number;
+    autoIndexSize: number;
+    chunkSize: number;
+    indexSize: number;
+  } = (() => {
+    if (trainingType === DatasetCollectionDataProcessModeEnum.qa) {
+      return {
+        autoChunkSize: getLLMDefaultChunkSize(agentModel),
+        autoIndexSize: 512,
+        chunkSize: qaChunkSize,
+        indexSize: 512
+      };
+    } else if (autoIndexes) {
+      return {
+        autoChunkSize: chunkAutoChunkSize,
+        autoIndexSize: getAutoIndexSize(vectorModel),
+        chunkSize: embeddingChunkSize,
+        indexSize
+      };
+    } else {
+      return {
+        autoChunkSize: chunkAutoChunkSize,
+        autoIndexSize: getAutoIndexSize(vectorModel),
+        chunkSize: embeddingChunkSize,
+        indexSize
+      };
+    }
+  })();
+
+  const { chunkSize: formatChunkIndex, indexSize: formatIndexSize } = (() => {
+    if (chunkSettingMode === ChunkSettingModeEnum.auto) {
+      return {
+        chunkSize: trainingModeSize.autoChunkSize,
+        indexSize: trainingModeSize.autoIndexSize
+      };
+    } else {
+      return {
+        chunkSize: trainingModeSize.chunkSize,
+        indexSize: trainingModeSize.indexSize
+      };
+    }
+  })();
+
+  return {
+    trainingType,
+    imageIndex,
+    autoIndexes,
+
+    chunkSettingMode,
+    chunkSplitMode,
+
+    chunkSize: formatChunkIndex,
+    indexSize: formatIndexSize,
+
+    chunkSplitter,
+    qaPrompt: trainingType === DatasetCollectionDataProcessModeEnum.qa ? qaPrompt : undefined
+  };
+};

projects/app/src/pageComponents/dataset/detail/Import/Context.tsx

@@ -25,6 +25,14 @@ import {
   getAutoIndexSize,
   getMaxIndexSize
 } from '@fastgpt/global/core/dataset/training/utils';
+import { CollectionChunkFormType } from '../Form/CollectionChunkForm';
+
+type ChunkSizeFieldType = 'embeddingChunkSize' | 'qaChunkSize';
+export type ImportFormType = {
+  customPdfParse: boolean;
+
+  webSelector: string;
+} & CollectionChunkFormType;
 
 type TrainingFiledType = {
   chunkOverlapRatio: number;
@@ -51,26 +59,6 @@ type DatasetImportContextType = {
   setSources: React.Dispatch<React.SetStateAction<ImportSourceItemType[]>>;
 } & TrainingFiledType;
 
-type ChunkSizeFieldType = 'embeddingChunkSize' | 'qaChunkSize';
-export type ImportFormType = {
-  customPdfParse: boolean;
-
-  trainingType: DatasetCollectionDataProcessModeEnum;
-  imageIndex: boolean;
-  autoIndexes: boolean;
-
-  chunkSettingMode: ChunkSettingModeEnum;
-
-  chunkSplitMode: DataChunkSplitModeEnum;
-  embeddingChunkSize: number;
-  qaChunkSize: number;
-  chunkSplitter: string;
-  indexSize: number;
-
-  qaPrompt: string;
-  webSelector: string;
-};
-
 export const DatasetImportContext = createContext<DatasetImportContextType>({
   importSource: ImportDataSourceEnum.fileLocal,
   goToNext: function (): void {
@@ -314,14 +302,7 @@ const DatasetImportContextProvider = ({ children }: { children: React.ReactNode
         chunkSplitter
       };
     }
-  }, [
-    chunkSettingMode,
-    TrainingModeMap.autoChunkSize,
-    TrainingModeMap.autoIndexSize,
-    TrainingModeMap.chunkSize,
-    TrainingModeMap.indexSize,
-    chunkSplitter
-  ]);
+  }, [chunkSettingMode, TrainingModeMap, chunkSplitter]);
 
   const contextValue = {
     ...TrainingModeMap,

projects/app/src/pageComponents/dataset/detail/Import/commonProgress/DataProcess.tsx

@@ -1,13 +1,8 @@
-import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
+import React, { useCallback } from 'react';
 import {
   Box,
   Flex,
-  Input,
   Button,
-  ModalBody,
-  ModalFooter,
-  Textarea,
-  useDisclosure,
   Checkbox,
   Accordion,
   AccordionItem,
@@ -16,93 +11,26 @@ import {
   AccordionIcon,
   HStack
 } from '@chakra-ui/react';
-import MyIcon from '@fastgpt/web/components/common/Icon';
 import { useTranslation } from 'next-i18next';
-import LeftRadio from '@fastgpt/web/components/common/Radio/LeftRadio';
-import {
-  DataChunkSplitModeEnum,
-  DatasetCollectionDataProcessModeEnum,
-  DatasetCollectionDataProcessModeMap
-} from '@fastgpt/global/core/dataset/constants';
-import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
-import MyTooltip from '@fastgpt/web/components/common/MyTooltip';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
-import MyModal from '@fastgpt/web/components/common/MyModal';
-import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
 import MyTag from '@fastgpt/web/components/common/Tag/index';
 import { useContextSelector } from 'use-context-selector';
 import { DatasetImportContext } from '../Context';
 import FormLabel from '@fastgpt/web/components/common/MyBox/FormLabel';
-import MyNumberInput from '@fastgpt/web/components/common/Input/NumberInput';
 import QuestionTip from '@fastgpt/web/components/common/MyTooltip/QuestionTip';
 import { shadowLight } from '@fastgpt/web/styles/theme';
-import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
-import MySelect from '@fastgpt/web/components/common/MySelect';
-import { getIndexSizeSelectList } from '@fastgpt/global/core/dataset/training/utils';
-import RadioGroup from '@fastgpt/web/components/common/Radio/RadioGroup';
+import CollectionChunkForm from '../../Form/CollectionChunkForm';
+import { DatasetCollectionDataProcessModeEnum } from '@fastgpt/global/core/dataset/constants';
 
 function DataProcess() {
   const { t } = useTranslation();
   const { feConfigs } = useSystemStore();
 
-  const {
-    goToNext,
-    processParamsForm,
-    chunkSizeField,
-    minChunkSize,
-    maxChunkSize,
-    maxIndexSize,
-    indexSize
-  } = useContextSelector(DatasetImportContext, (v) => v);
-  const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
-  const { setValue, register, watch, getValues } = processParamsForm;
-
-  const trainingType = watch('trainingType');
-  const trainingModeList = useMemo(() => {
-    const list = Object.entries(DatasetCollectionDataProcessModeMap);
-    return list
-      .filter(([key]) => key !== DatasetCollectionDataProcessModeEnum.auto)
-      .map(([key, value]) => ({
-        title: t(value.label as any),
-        value: key as DatasetCollectionDataProcessModeEnum,
-        tooltip: t(value.tooltip as any)
-      }));
-  }, [t]);
-
-  const chunkSettingMode = watch('chunkSettingMode');
-  const chunkSplitMode = watch('chunkSplitMode');
-
-  const customSplitList = [
-    { label: t('dataset:split_sign_null'), value: '' },
-    { label: t('dataset:split_sign_break'), value: '\\n' },
-    { label: t('dataset:split_sign_break2'), value: '\\n\\n' },
-    { label: t('dataset:split_sign_period'), value: '.|。' },
-    { label: t('dataset:split_sign_exclamatiob'), value: '!|！' },
-    { label: t('dataset:split_sign_question'), value: '?|？' },
-    { label: t('dataset:split_sign_semicolon'), value: ';|；' },
-    { label: '=====', value: '=====' },
-    { label: t('dataset:split_sign_custom'), value: 'Other' }
-  ];
-
-  const [customListSelectValue, setCustomListSelectValue] = useState(getValues('chunkSplitter'));
-  useEffect(() => {
-    if (customListSelectValue === 'Other') {
-      setValue('chunkSplitter', '');
-    } else {
-      setValue('chunkSplitter', customListSelectValue);
-    }
-  }, [customListSelectValue, setValue]);
-
-  // Index size
-  const indexSizeSeletorList = useMemo(() => getIndexSizeSelectList(maxIndexSize), [maxIndexSize]);
-
-  // QA
-  const qaPrompt = watch('qaPrompt');
-  const {
-    isOpen: isOpenCustomPrompt,
-    onOpen: onOpenCustomPrompt,
-    onClose: onCloseCustomPrompt
-  } = useDisclosure();
+  const { goToNext, processParamsForm, chunkSize } = useContextSelector(
+    DatasetImportContext,
+    (v) => v
+  );
+  const { register } = processParamsForm;
 
   const Title = useCallback(({ title }: { title: string }) => {
     return (
@@ -116,16 +44,7 @@ function DataProcess() {
     );
   }, []);
 
-  // Adapt auto training
-  useEffect(() => {
-    if (trainingType === DatasetCollectionDataProcessModeEnum.auto) {
-      setValue('autoIndexes', true);
-      setValue('trainingType', DatasetCollectionDataProcessModeEnum.chunk);
-    }
-  }, [trainingType, setValue]);
-
   const showFileParseSetting = feConfigs?.showCustomPdfParse;
-  const showQAPromptInput = trainingType === DatasetCollectionDataProcessModeEnum.qa;
 
   return (
     <>
@@ -179,238 +98,8 @@ function DataProcess() {
             <Title title={t('dataset:import_data_process_setting')} />
 
             <AccordionPanel p={2}>
-              <Box mt={2}>
-                <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
-                  {t('dataset:training_mode')}
-                </Box>
-                <LeftRadio<DatasetCollectionDataProcessModeEnum>
-                  list={trainingModeList}
-                  px={3}
-                  py={2.5}
-                  value={trainingType}
-                  onChange={(e) => {
-                    setValue('trainingType', e);
-                  }}
-                  defaultBg="white"
-                  activeBg="white"
-                  gridTemplateColumns={'repeat(2, 1fr)'}
-                />
-              </Box>
-              {trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
-                <Box mt={6}>
-                  <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
-                    {t('dataset:enhanced_indexes')}
-                  </Box>
-                  <HStack gap={[3, 7]}>
-                    <HStack flex={'1'} spacing={1}>
-                      <MyTooltip
-                        label={!feConfigs?.isPlus ? t('common:commercial_function_tip') : ''}
-                      >
-                        <Checkbox isDisabled={!feConfigs?.isPlus} {...register('autoIndexes')}>
-                          <FormLabel>{t('dataset:auto_indexes')}</FormLabel>
-                        </Checkbox>
-                      </MyTooltip>
-                      <QuestionTip label={t('dataset:auto_indexes_tips')} />
-                    </HStack>
-                    <HStack flex={'1'} spacing={1}>
-                      <MyTooltip
-                        label={
-                          !feConfigs?.isPlus
-                            ? t('common:commercial_function_tip')
-                            : !datasetDetail?.vlmModel
-                              ? t('common:error_vlm_not_config')
-                              : ''
-                        }
-                      >
-                        <Checkbox
-                          isDisabled={!feConfigs?.isPlus || !datasetDetail?.vlmModel}
-                          {...register('imageIndex')}
-                        >
-                          <FormLabel>{t('dataset:image_auto_parse')}</FormLabel>
-                        </Checkbox>
-                      </MyTooltip>
-                      <QuestionTip label={t('dataset:image_auto_parse_tips')} />
-                    </HStack>
-                  </HStack>
-                </Box>
-              )}
-              <Box mt={6}>
-                <Box fontSize={'sm'} mb={2} color={'myGray.600'}>
-                  {t('dataset:params_setting')}
-                </Box>
-                <LeftRadio<ChunkSettingModeEnum>
-                  list={[
-                    {
-                      title: t('dataset:default_params'),
-                      desc: t('dataset:default_params_desc'),
-                      value: ChunkSettingModeEnum.auto
-                    },
-                    {
-                      title: t('dataset:custom_data_process_params'),
-                      desc: t('dataset:custom_data_process_params_desc'),
-                      value: ChunkSettingModeEnum.custom,
-                      children: chunkSettingMode === ChunkSettingModeEnum.custom && (
-                        <Box mt={5}>
-                          <Box>
-                            <RadioGroup<DataChunkSplitModeEnum>
-                              list={[
-                                {
-                                  title: t('dataset:split_chunk_size'),
-                                  value: DataChunkSplitModeEnum.size
-                                },
-                                {
-                                  title: t('dataset:split_chunk_char'),
-                                  value: DataChunkSplitModeEnum.char,
-                                  tooltip: t('dataset:custom_split_sign_tip')
-                                }
-                              ]}
-                              value={chunkSplitMode}
-                              onChange={(e) => {
-                                setValue('chunkSplitMode', e);
-                              }}
-                            />
-
-                            {chunkSplitMode === DataChunkSplitModeEnum.size && (
-                              <Box
-                                mt={1.5}
-                                css={{
-                                  '& > span': {
-                                    display: 'block'
-                                  }
-                                }}
-                              >
-                                <MyTooltip
-                                  label={t('common:core.dataset.import.Chunk Range', {
-                                    min: minChunkSize,
-                                    max: maxChunkSize
-                                  })}
-                                >
-                                  <MyNumberInput
-                                    register={register}
-                                    name={chunkSizeField}
-                                    min={minChunkSize}
-                                    max={maxChunkSize}
-                                    size={'sm'}
-                                    step={100}
-                                  />
-                                </MyTooltip>
-                              </Box>
-                            )}
-
-                            {chunkSplitMode === DataChunkSplitModeEnum.char && (
-                              <HStack mt={1.5}>
-                                <Box flex={'1 0 0'}>
-                                  <MySelect<string>
-                                    list={customSplitList}
-                                    size={'sm'}
-                                    bg={'myGray.50'}
-                                    value={customListSelectValue}
-                                    h={'32px'}
-                                    onChange={(val) => {
-                                      setCustomListSelectValue(val);
-                                    }}
-                                  />
-                                </Box>
-                                {customListSelectValue === 'Other' && (
-                                  <Input
-                                    flex={'1 0 0'}
-                                    h={'32px'}
-                                    size={'sm'}
-                                    bg={'myGray.50'}
-                                    placeholder="\n;======;==SPLIT=="
-                                    {...register('chunkSplitter')}
-                                  />
-                                )}
-                              </HStack>
-                            )}
-                          </Box>
-
-                          {trainingType === DatasetCollectionDataProcessModeEnum.chunk && (
-                            <Box>
-                              <Flex alignItems={'center'} mt={3}>
-                                <Box>{t('dataset:index_size')}</Box>
-                                <QuestionTip label={t('dataset:index_size_tips')} />
-                              </Flex>
-                              <Box mt={1}>
-                                <MySelect<number>
-                                  bg={'myGray.50'}
-                                  list={indexSizeSeletorList}
-                                  value={indexSize}
-                                  onChange={(val) => {
-                                    setValue('indexSize', val);
-                                  }}
-                                />
-                              </Box>
-                            </Box>
-                          )}
-
-                          {showQAPromptInput && (
-                            <Box mt={3}>
-                              <Box>{t('common:core.dataset.collection.QA Prompt')}</Box>
-                              <Box
-                                position={'relative'}
-                                py={2}
-                                px={3}
-                                bg={'myGray.50'}
-                                fontSize={'xs'}
-                                whiteSpace={'pre-wrap'}
-                                border={'1px'}
-                                borderColor={'borderColor.base'}
-                                borderRadius={'md'}
-                                maxH={'140px'}
-                                overflow={'auto'}
-                                _hover={{
-                                  '& .mask': {
-                                    display: 'block'
-                                  }
-                                }}
-                              >
-                                {qaPrompt}
-
-                                <Box
-                                  display={'none'}
-                                  className="mask"
-                                  position={'absolute'}
-                                  top={0}
-                                  right={0}
-                                  bottom={0}
-                                  left={0}
-                                  background={
-                                    'linear-gradient(182deg, rgba(255, 255, 255, 0.00) 1.76%, #FFF 84.07%)'
-                                  }
-                                >
-                                  <Button
-                                    size="xs"
-                                    variant={'whiteBase'}
-                                    leftIcon={<MyIcon name={'edit'} w={'13px'} />}
-                                    color={'black'}
-                                    position={'absolute'}
-                                    right={2}
-                                    bottom={2}
-                                    onClick={onOpenCustomPrompt}
-                                  >
-                                    {t('common:core.dataset.import.Custom prompt')}
-                                  </Button>
-                                </Box>
-                              </Box>
-                            </Box>
-                          )}
-                        </Box>
-                      )
-                    }
-                  ]}
-                  gridGap={3}
-                  px={3}
-                  py={3}
-                  defaultBg="white"
-                  activeBg="white"
-                  value={chunkSettingMode}
-                  w={'100%'}
-                  onChange={(e) => {
-                    setValue('chunkSettingMode', e);
-                  }}
-                />
-              </Box>
+              {/* @ts-ignore */}
+              <CollectionChunkForm form={processParamsForm} />
             </AccordionPanel>
           </AccordionItem>
 
@@ -425,57 +114,8 @@ function DataProcess() {
           </Flex>
         </Accordion>
       </Box>
-
-      {isOpenCustomPrompt && (
-        <PromptTextarea
-          defaultValue={qaPrompt}
-          onChange={(e) => {
-            setValue('qaPrompt', e);
-          }}
-          onClose={onCloseCustomPrompt}
-        />
-      )}
     </>
   );
 }
 
 export default React.memo(DataProcess);
-
-const PromptTextarea = ({
-  defaultValue,
-  onChange,
-  onClose
-}: {
-  defaultValue: string;
-  onChange: (e: string) => void;
-  onClose: () => void;
-}) => {
-  const ref = useRef<HTMLTextAreaElement>(null);
-  const { t } = useTranslation();
-
-  return (
-    <MyModal
-      isOpen
-      title={t('common:core.dataset.import.Custom prompt')}
-      iconSrc="modal/edit"
-      w={'600px'}
-      onClose={onClose}
-    >
-      <ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
-        <Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
-        <Box>{Prompt_AgentQA.fixedText}</Box>
-      </ModalBody>
-      <ModalFooter>
-        <Button
-          onClick={() => {
-            const val = ref.current?.value || Prompt_AgentQA.description;
-            onChange(val);
-            onClose();
-          }}
-        >
-          {t('common:common.Confirm')}
-        </Button>
-      </ModalFooter>
-    </MyModal>
-  );
-};

projects/app/src/pageComponents/dataset/detail/MetaDataCard.tsx

@@ -85,9 +85,13 @@ const MetaDataCard = ({ datasetId }: { datasetId: string }) => {
         value: t(DatasetCollectionDataProcessModeMap[collection.trainingType]?.label as any)
       },
       {
-        label: t('common:core.dataset.collection.metadata.Chunk Size'),
+        label: t('dataset:chunk_size'),
         value: collection.chunkSize || '-'
       },
+      {
+        label: t('dataset:index_size'),
+        value: collection.indexSize || '-'
+      },
       ...(webSelector
         ? [
             {

projects/app/src/pages/api/admin/clearInvalidData.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { addHours } from 'date-fns';
 import { MongoImage } from '@fastgpt/service/common/file/image/schema';
@@ -56,7 +55,6 @@ async function checkInvalidImg(start: Date, end: Date, limit = 50) {
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
     const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number };
 

projects/app/src/pages/api/admin/initv47.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoPlugin } from '@fastgpt/service/core/plugin/schema';
 import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
@@ -8,7 +7,6 @@ import { PluginTypeEnum } from '@fastgpt/global/core/plugin/constants';
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     await MongoPlugin.updateMany(

projects/app/src/pages/api/admin/initv471.ts

@@ -1,13 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
 
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     // 删除索引

projects/app/src/pages/api/admin/initv4810.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoAppVersion } from '@fastgpt/service/core/app/version/schema';
 import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
@@ -9,7 +8,6 @@ import { POST } from '@fastgpt/service/common/api/plusRequest';
 /* 初始化发布的版本 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     await MongoAppVersion.updateMany(

projects/app/src/pages/api/admin/initv4823.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { addHours } from 'date-fns';
 import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
@@ -174,7 +173,6 @@ const checkInvalidDataText = async () => {
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
     const { start = -2, end = -360 * 24 } = req.body as { start: number; end: number };
 

projects/app/src/pages/api/admin/initv486.ts

@@ -1,14 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { PgClient } from '@fastgpt/service/common/vectorStore/pg';
 import { MongoApp } from '@fastgpt/service/core/app/schema';
 
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     await MongoApp.updateMany(

projects/app/src/pages/api/admin/initv488.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
 import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/dataset/constant';
@@ -8,7 +7,6 @@ import { DatasetDefaultPermissionVal } from '@fastgpt/global/support/permission/
 /* pg 中的数据搬到 mongo dataset.datas 中，并做映射 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     await authCert({ req, authRoot: true });
 
     await MongoDataset.updateMany(

projects/app/src/pages/api/admin/initv494.ts

@@ -0,0 +1,53 @@
+import { NextAPI } from '@/service/middleware/entry';
+import { retryFn } from '@fastgpt/global/common/system/utils';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { upsertWebsiteSyncJobScheduler } from '@fastgpt/service/core/dataset/websiteSync';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { addHours } from 'date-fns';
+import { NextApiRequest, NextApiResponse } from 'next';
+
+const initWebsiteSyncData = async () => {
+  // find out all website dataset
+  const datasets = await MongoDataset.find({ type: DatasetTypeEnum.websiteDataset }).lean();
+
+  console.log('更新站点同步的定时器');
+  // Add scheduler for all website dataset
+  await Promise.all(
+    datasets.map((dataset) => {
+      if (dataset.autoSync) {
+        // 随机生成一个往后 1～24 小时的时间
+        const time = addHours(new Date(), Math.floor(Math.random() * 23) + 1);
+        return retryFn(() =>
+          upsertWebsiteSyncJobScheduler({ datasetId: String(dataset._id) }, time.getTime())
+        );
+      }
+    })
+  );
+
+  console.log('移除站点同步集合的定时器');
+  // Remove all nextSyncTime
+  await retryFn(() =>
+    MongoDatasetCollection.updateMany(
+      {
+        teamId: datasets.map((dataset) => dataset.teamId),
+        datasetId: datasets.map((dataset) => dataset._id)
+      },
+      {
+        $unset: {
+          nextSyncTime: 1
+        }
+      }
+    )
+  );
+};
+async function handler(req: NextApiRequest, _res: NextApiResponse) {
+  await authCert({ req, authRoot: true });
+
+  await initWebsiteSyncData();
+
+  return { success: true };
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/common/file/read.ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authFileToken } from '@fastgpt/service/support/permission/controller';
 import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
@@ -23,8 +22,6 @@ const previewableExtensions = [
 // Abandoned, use: file/read/[filename].ts
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
-
     const { token } = req.query as { token: string };
 
     const { fileId, bucketName } = await authFileToken(token);

projects/app/src/pages/api/common/file/read/[filename].ts

@@ -1,6 +1,5 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
 import { authFileToken } from '@fastgpt/service/support/permission/controller';
 import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
@@ -21,8 +20,6 @@ const previewableExtensions = [
 ];
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
-
     const { token, filename } = req.query as { token: string; filename: string };
 
     const { fileId, bucketName } = await authFileToken(token);

projects/app/src/pages/api/common/file/uploadImage.ts

@@ -1,5 +1,4 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
-import { connectToDatabase } from '@/service/mongo';
 import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller';
 import { UploadImgProps } from '@fastgpt/global/common/file/api';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
@@ -9,7 +8,6 @@ import { NextAPI } from '@/service/middleware/entry';
   Upload avatar image
 */
 async function handler(req: NextApiRequest, res: NextApiResponse): Promise<string> {
-  await connectToDatabase();
   const body = req.body as UploadImgProps;
 
   const { teamId } = await authCert({ req, authToken: true });

projects/app/src/pages/api/common/system/unlockTask.ts

@@ -1,12 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { startTrainingQueue } from '@/service/core/dataset/training/utils';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
     await authCert({ req, authToken: true });
     startTrainingQueue();
   } catch (error) {}

projects/app/src/pages/api/common/tools/urlFetch.ts

@@ -2,13 +2,12 @@
 import { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
-import { connectToDatabase } from '@/service/mongo';
+
 import { UrlFetchParams, UrlFetchResponse } from '@fastgpt/global/common/file/api.d';
 import { urlsFetch } from '@fastgpt/service/common/string/cheerio';
 
 const fetchContent = async (req: NextApiRequest, res: NextApiResponse) => {
   try {
-    await connectToDatabase();
     let { urlList = [], selector } = req.body as UrlFetchParams;
 
     if (!urlList || urlList.length === 0) {

projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts

@@ -1,6 +1,6 @@
 import type { NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { pushQuestionGuideUsage } from '@/service/support/wallet/usage/push';
 import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
 import { ApiRequestProps } from '@fastgpt/service/type/next';
@@ -27,7 +27,6 @@ async function handler(
   res: NextApiResponse<any>
 ) {
   try {
-    await connectToDatabase();
     const { messages } = req.body;
 
     const { tmbId, teamId } = await authChatCert({

projects/app/src/pages/api/core/chat/chatTest.ts

@@ -182,7 +182,8 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
       histories: newHistories,
       stream: true,
       maxRunTimes: WORKFLOW_MAX_RUN_TIMES,
-      workflowStreamResponse: workflowResponseWrite
+      workflowStreamResponse: workflowResponseWrite,
+      version: 'v2'
     });
 
     workflowResponseWrite({
@@ -197,11 +198,6 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
       event: SseResponseEventEnum.answer,
       data: '[DONE]'
     });
-    responseWrite({
-      res,
-      event: SseResponseEventEnum.flowResponses,
-      data: JSON.stringify(flowResponses)
-    });
 
     // save chat
     const isInteractiveRequest = !!getLastInteractiveValue(histories);

projects/app/src/pages/api/core/chat/feedback/adminUpdate.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import type { AdminUpdateFeedbackParams } from '@/global/core/chat/api.d';
 import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
 import { authChatCrud } from '@/service/support/permission/auth/chat';
@@ -8,7 +8,6 @@ import { authChatCrud } from '@/service/support/permission/auth/chat';
 /* 初始化我的聊天框，需要身份验证 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { appId, chatId, dataId, datasetId, feedbackDataId, q, a } =
       req.body as AdminUpdateFeedbackParams;
 

projects/app/src/pages/api/core/chat/feedback/closeCustom.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import type { CloseCustomFeedbackParams } from '@/global/core/chat/api.d';
 import { MongoChatItem } from '@fastgpt/service/core/chat/chatItemSchema';
@@ -10,7 +10,6 @@ import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
 /* remove custom feedback */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { appId, chatId, dataId, index } = req.body as CloseCustomFeedbackParams;
 
     if (!dataId || !appId || !chatId) {

projects/app/src/pages/api/core/chat/item/getSpeech.ts

@@ -1,6 +1,6 @@
 import type { NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { GetChatSpeechProps } from '@/global/core/chat/api.d';
 import { text2Speech } from '@fastgpt/service/core/ai/audio/speech';
 import { pushAudioSpeechUsage } from '@/service/support/wallet/usage/push';
@@ -17,7 +17,6 @@ import { ApiRequestProps } from '@fastgpt/service/type/next';
 */
 async function handler(req: ApiRequestProps<GetChatSpeechProps>, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { ttsConfig, input } = req.body;
 
     if (!ttsConfig.model || !ttsConfig.voice) {

projects/app/src/pages/api/core/dataset/collection/detail.ts

@@ -12,6 +12,9 @@ import { DatasetCollectionItemType } from '@fastgpt/global/core/dataset/type';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { collectionTagsToTagLabel } from '@fastgpt/service/core/dataset/collection/utils';
 import { getVectorCountByCollectionId } from '@fastgpt/service/common/vectorStore/controller';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { Types } from 'mongoose';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
 
 async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType> {
   const { id } = req.query as { id: string };
@@ -30,11 +33,21 @@ async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType>
   });
 
   // get file
-  const [file, indexAmount] = await Promise.all([
+  const [file, indexAmount, errorCount] = await Promise.all([
     collection?.fileId
       ? await getFileById({ bucketName: BucketNameEnum.dataset, fileId: collection.fileId })
       : undefined,
-    getVectorCountByCollectionId(collection.teamId, collection.datasetId, collection._id)
+    getVectorCountByCollectionId(collection.teamId, collection.datasetId, collection._id),
+    MongoDatasetTraining.countDocuments(
+      {
+        teamId: collection.teamId,
+        datasetId: collection.datasetId,
+        collectionId: id,
+        errorMsg: { $exists: true },
+        retryCount: { $lte: 0 }
+      },
+      readFromSecondary
+    )
   ]);
 
   return {
@@ -46,7 +59,8 @@ async function handler(req: NextApiRequest): Promise<DatasetCollectionItemType>
       tags: collection.tags
     }),
     permission,
-    file
+    file,
+    errorCount
   };
 }
 

projects/app/src/pages/api/core/dataset/collection/listV2.ts

@@ -93,6 +93,7 @@ async function handler(
           dataAmount: 0,
           indexAmount: 0,
           trainingAmount: 0,
+          hasError: false,
           permission
         }))
       ),
@@ -113,7 +114,7 @@ async function handler(
 
   // Compute data amount
   const [trainingAmount, dataAmount]: [
-    { _id: string; count: number }[],
+    { _id: string; count: number; hasError: boolean }[],
     { _id: string; count: number }[]
   ] = await Promise.all([
     MongoDatasetTraining.aggregate(
@@ -128,7 +129,8 @@ async function handler(
         {
           $group: {
             _id: '$collectionId',
-            count: { $sum: 1 }
+            count: { $sum: 1 },
+            hasError: { $max: { $cond: [{ $ifNull: ['$errorMsg', false] }, true, false] } }
           }
         }
       ],
@@ -168,6 +170,7 @@ async function handler(
       trainingAmount:
         trainingAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
       dataAmount: dataAmount.find((amount) => String(amount._id) === String(item._id))?.count || 0,
+      hasError: trainingAmount.find((amount) => String(amount._id) === String(item._id))?.hasError,
       permission
     }))
   );

projects/app/src/pages/api/core/dataset/collection/trainingDetail.ts

@@ -0,0 +1,170 @@
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import {
+  DatasetCollectionDataProcessModeEnum,
+  TrainingModeEnum
+} from '@fastgpt/global/core/dataset/constants';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
+import { NextAPI } from '@/service/middleware/entry';
+import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
+import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
+
+type getTrainingDetailParams = {
+  collectionId: string;
+};
+
+export type getTrainingDetailResponse = {
+  trainingType: DatasetCollectionDataProcessModeEnum;
+  advancedTraining: {
+    customPdfParse: boolean;
+    imageIndex: boolean;
+    autoIndexes: boolean;
+  };
+  queuedCounts: Record<TrainingModeEnum, number>;
+  trainingCounts: Record<TrainingModeEnum, number>;
+  errorCounts: Record<TrainingModeEnum, number>;
+  trainedCount: number;
+};
+
+const defaultCounts: Record<TrainingModeEnum, number> = {
+  qa: 0,
+  chunk: 0,
+  image: 0,
+  auto: 0
+};
+
+async function handler(
+  req: ApiRequestProps<{}, getTrainingDetailParams>
+): Promise<getTrainingDetailResponse> {
+  const { collectionId } = req.query;
+
+  const { collection } = await authDatasetCollection({
+    req,
+    authToken: true,
+    collectionId: collectionId as string,
+    per: ReadPermissionVal
+  });
+
+  const match = {
+    teamId: collection.teamId,
+    datasetId: collection.datasetId,
+    collectionId: collection._id
+  };
+
+  // Computed global queue
+  const minId = (
+    await MongoDatasetTraining.findOne(
+      {
+        teamId: collection.teamId,
+        datasetId: collection.datasetId,
+        collectionId: collection._id
+      },
+      { sort: { _id: 1 }, select: '_id' },
+      readFromSecondary
+    ).lean()
+  )?._id;
+
+  const [ququedCountData, trainingCountData, errorCountData, trainedCount] = (await Promise.all([
+    minId
+      ? MongoDatasetTraining.aggregate(
+          [
+            {
+              $match: {
+                _id: { $lt: minId },
+                retryCount: { $gt: 0 },
+                lockTime: { $lt: new Date('2050/1/1') }
+              }
+            },
+            {
+              $group: {
+                _id: '$mode',
+                count: { $sum: 1 }
+              }
+            }
+          ],
+          readFromSecondary
+        )
+      : Promise.resolve([]),
+    MongoDatasetTraining.aggregate(
+      [
+        {
+          $match: {
+            ...match,
+            retryCount: { $gt: 0 },
+            lockTime: { $lt: new Date('2050/1/1') }
+          }
+        },
+        {
+          $group: {
+            _id: '$mode',
+            count: { $sum: 1 }
+          }
+        }
+      ],
+      readFromSecondary
+    ),
+    MongoDatasetTraining.aggregate(
+      [
+        {
+          $match: {
+            ...match,
+            retryCount: { $lte: 0 },
+            errorMsg: { $exists: true }
+          }
+        },
+        {
+          $group: {
+            _id: '$mode',
+            count: { $sum: 1 }
+          }
+        }
+      ],
+      readFromSecondary
+    ),
+    MongoDatasetData.countDocuments(match, readFromSecondary)
+  ])) as [
+    { _id: TrainingModeEnum; count: number }[],
+    { _id: TrainingModeEnum; count: number }[],
+    { _id: TrainingModeEnum; count: number }[],
+    number
+  ];
+
+  const queuedCounts = ququedCountData.reduce(
+    (acc, item) => {
+      acc[item._id] = item.count;
+      return acc;
+    },
+    { ...defaultCounts }
+  );
+  const trainingCounts = trainingCountData.reduce(
+    (acc, item) => {
+      acc[item._id] = item.count;
+      return acc;
+    },
+    { ...defaultCounts }
+  );
+  const errorCounts = errorCountData.reduce(
+    (acc, item) => {
+      acc[item._id] = item.count;
+      return acc;
+    },
+    { ...defaultCounts }
+  );
+
+  return {
+    trainingType: collection.trainingType,
+    advancedTraining: {
+      customPdfParse: !!collection.customPdfParse,
+      imageIndex: !!collection.imageIndex,
+      autoIndexes: !!collection.autoIndexes
+    },
+
+    queuedCounts,
+    trainingCounts,
+    errorCounts,
+    trainedCount
+  };
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/delete.ts

@@ -9,6 +9,8 @@ import { OwnerPermissionVal } from '@fastgpt/global/support/permission/constant'
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
 import { MongoDatasetCollectionTags } from '@fastgpt/service/core/dataset/tag/schema';
 import { removeImageByPath } from '@fastgpt/service/common/file/image/controller';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
+import { removeWebsiteSyncJobScheduler } from '@fastgpt/service/core/dataset/websiteSync';
 
 async function handler(req: NextApiRequest) {
   const { id: datasetId } = req.query as {
@@ -40,6 +42,13 @@ async function handler(req: NextApiRequest) {
     datasetId: { $in: datasetIds }
   });
 
+  await Promise.all(
+    datasets.map((dataset) => {
+      if (dataset.type === DatasetTypeEnum.websiteDataset)
+        return removeWebsiteSyncJobScheduler(String(dataset._id));
+    })
+  );
+
   // delete all dataset.data and pg data
   await mongoSessionRun(async (session) => {
     // delete dataset data

projects/app/src/pages/api/core/dataset/detail.ts

@@ -5,6 +5,8 @@ import { NextAPI } from '@/service/middleware/entry';
 import { DatasetItemType } from '@fastgpt/global/core/dataset/type';
 import { ApiRequestProps } from '@fastgpt/service/type/next';
 import { CommonErrEnum } from '@fastgpt/global/common/error/code/common';
+import { getWebsiteSyncDatasetStatus } from '@fastgpt/service/core/dataset/websiteSync';
+import { DatasetStatusEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constants';
 
 type Query = {
   id: string;
@@ -28,8 +30,21 @@ async function handler(req: ApiRequestProps<Query>): Promise<DatasetItemType> {
     per: ReadPermissionVal
   });
 
+  const { status, errorMsg } = await (async () => {
+    if (dataset.type === DatasetTypeEnum.websiteDataset) {
+      return await getWebsiteSyncDatasetStatus(datasetId);
+    }
+
+    return {
+      status: DatasetStatusEnum.active,
+      errorMsg: undefined
+    };
+  })();
+
   return {
     ...dataset,
+    status,
+    errorMsg,
     apiServer: dataset.apiServer
       ? {
           baseUrl: dataset.apiServer.baseUrl,

projects/app/src/pages/api/core/dataset/training/deleteTrainingData.ts

@@ -0,0 +1,39 @@
+import { ManagePermissionVal } from '@fastgpt/global/support/permission/constant';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
+import { NextAPI } from '@/service/middleware/entry';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
+
+export type deleteTrainingDataBody = {
+  datasetId: string;
+  collectionId: string;
+  dataId: string;
+};
+
+export type deleteTrainingDataQuery = {};
+
+export type deleteTrainingDataResponse = {};
+
+async function handler(
+  req: ApiRequestProps<deleteTrainingDataBody, deleteTrainingDataQuery>
+): Promise<deleteTrainingDataResponse> {
+  const { datasetId, collectionId, dataId } = req.body;
+
+  const { teamId } = await authDatasetCollection({
+    req,
+    authToken: true,
+    authApiKey: true,
+    collectionId,
+    per: ManagePermissionVal
+  });
+
+  await MongoDatasetTraining.deleteOne({
+    teamId,
+    datasetId,
+    _id: dataId
+  });
+
+  return {};
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/training/getTrainingDataDetail.ts

@@ -0,0 +1,52 @@
+import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
+import { NextAPI } from '@/service/middleware/entry';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
+
+export type getTrainingDataDetailQuery = {};
+
+export type getTrainingDataDetailBody = {
+  datasetId: string;
+  collectionId: string;
+  dataId: string;
+};
+
+export type getTrainingDataDetailResponse =
+  | {
+      _id: string;
+      datasetId: string;
+      mode: string;
+      q: string;
+      a: string;
+    }
+  | undefined;
+
+async function handler(
+  req: ApiRequestProps<getTrainingDataDetailBody, getTrainingDataDetailQuery>
+): Promise<getTrainingDataDetailResponse> {
+  const { datasetId, collectionId, dataId } = req.body;
+
+  const { teamId } = await authDatasetCollection({
+    req,
+    authToken: true,
+    collectionId,
+    per: ReadPermissionVal
+  });
+
+  const data = await MongoDatasetTraining.findOne({ teamId, datasetId, _id: dataId }).lean();
+
+  if (!data) {
+    return undefined;
+  }
+
+  return {
+    _id: data._id,
+    datasetId: data.datasetId,
+    mode: data.mode,
+    q: data.q,
+    a: data.a
+  };
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/training/getTrainingError.ts

@@ -0,0 +1,51 @@
+import { NextAPI } from '@/service/middleware/entry';
+import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
+import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { parsePaginationRequest } from '@fastgpt/service/common/api/pagination';
+import { readFromSecondary } from '@fastgpt/service/common/mongo/utils';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
+import { PaginationProps, PaginationResponse } from '@fastgpt/web/common/fetch/type';
+
+export type getTrainingErrorBody = PaginationProps<{
+  collectionId: string;
+}>;
+
+export type getTrainingErrorResponse = PaginationResponse<DatasetTrainingSchemaType>;
+
+async function handler(req: ApiRequestProps<getTrainingErrorBody, {}>) {
+  const { collectionId } = req.body;
+  const { offset, pageSize } = parsePaginationRequest(req);
+
+  const { collection } = await authDatasetCollection({
+    req,
+    authToken: true,
+    collectionId,
+    per: ReadPermissionVal
+  });
+
+  const match = {
+    teamId: collection.teamId,
+    datasetId: collection.datasetId,
+    collectionId: collection._id,
+    errorMsg: { $exists: true }
+  };
+
+  const [errorList, total] = await Promise.all([
+    MongoDatasetTraining.find(match, undefined, {
+      ...readFromSecondary
+    })
+      .skip(offset)
+      .limit(pageSize)
+      .lean(),
+    MongoDatasetTraining.countDocuments(match, { ...readFromSecondary })
+  ]);
+
+  return {
+    list: errorList,
+    total
+  };
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/training/updateTrainingData.ts

@@ -0,0 +1,59 @@
+import { WritePermissionVal } from '@fastgpt/global/support/permission/constant';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authDatasetCollection } from '@fastgpt/service/support/permission/dataset/auth';
+import { NextAPI } from '@/service/middleware/entry';
+import { ApiRequestProps } from '@fastgpt/service/type/next';
+import { addMinutes } from 'date-fns';
+
+export type updateTrainingDataBody = {
+  datasetId: string;
+  collectionId: string;
+  dataId: string;
+  q?: string;
+  a?: string;
+  chunkIndex?: number;
+};
+
+export type updateTrainingDataQuery = {};
+
+export type updateTrainingDataResponse = {};
+
+async function handler(
+  req: ApiRequestProps<updateTrainingDataBody, updateTrainingDataQuery>
+): Promise<updateTrainingDataResponse> {
+  const { datasetId, collectionId, dataId, q, a, chunkIndex } = req.body;
+
+  const { teamId } = await authDatasetCollection({
+    req,
+    authToken: true,
+    authApiKey: true,
+    collectionId,
+    per: WritePermissionVal
+  });
+
+  const data = await MongoDatasetTraining.findOne({ teamId, datasetId, _id: dataId });
+
+  if (!data) {
+    return Promise.reject('data not found');
+  }
+
+  await MongoDatasetTraining.updateOne(
+    {
+      teamId,
+      datasetId,
+      _id: dataId
+    },
+    {
+      $unset: { errorMsg: '' },
+      retryCount: 3,
+      ...(q !== undefined && { q }),
+      ...(a !== undefined && { a }),
+      ...(chunkIndex !== undefined && { chunkIndex }),
+      lockTime: addMinutes(new Date(), -10)
+    }
+  );
+
+  return {};
+}
+
+export default NextAPI(handler);

projects/app/src/pages/api/core/dataset/update.ts

@@ -30,6 +30,13 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection
 import { addDays } from 'date-fns';
 import { refreshSourceAvatar } from '@fastgpt/service/common/file/image/controller';
 import { MongoResourcePermission } from '@fastgpt/service/support/permission/schema';
+import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
+import {
+  removeWebsiteSyncJobScheduler,
+  upsertWebsiteSyncJobScheduler
+} from '@fastgpt/service/core/dataset/websiteSync';
+import { delDatasetRelevantData } from '@fastgpt/service/core/dataset/controller';
+import { isEqual } from 'lodash';
 
 export type DatasetUpdateQuery = {};
 export type DatasetUpdateResponse = any;
@@ -62,8 +69,8 @@ async function handler(
     apiServer,
     yuqueServer,
     feishuServer,
-    status,
-    autoSync
+    autoSync,
+    chunkSettings
   } = req.body;
 
   if (!id) {
@@ -114,6 +121,39 @@ async function handler(
   });
 
   const onUpdate = async (session: ClientSession) => {
+    // Website dataset update chunkSettings, need to clean up dataset
+    if (
+      dataset.type === DatasetTypeEnum.websiteDataset &&
+      chunkSettings &&
+      dataset.chunkSettings &&
+      !isEqual(
+        {
+          imageIndex: dataset.chunkSettings.imageIndex,
+          autoIndexes: dataset.chunkSettings.autoIndexes,
+          trainingType: dataset.chunkSettings.trainingType,
+          chunkSettingMode: dataset.chunkSettings.chunkSettingMode,
+          chunkSplitMode: dataset.chunkSettings.chunkSplitMode,
+          chunkSize: dataset.chunkSettings.chunkSize,
+          chunkSplitter: dataset.chunkSettings.chunkSplitter,
+          indexSize: dataset.chunkSettings.indexSize,
+          qaPrompt: dataset.chunkSettings.qaPrompt
+        },
+        {
+          imageIndex: chunkSettings.imageIndex,
+          autoIndexes: chunkSettings.autoIndexes,
+          trainingType: chunkSettings.trainingType,
+          chunkSettingMode: chunkSettings.chunkSettingMode,
+          chunkSplitMode: chunkSettings.chunkSplitMode,
+          chunkSize: chunkSettings.chunkSize,
+          chunkSplitter: chunkSettings.chunkSplitter,
+          indexSize: chunkSettings.indexSize,
+          qaPrompt: chunkSettings.qaPrompt
+        }
+      )
+    ) {
+      await delDatasetRelevantData({ datasets: [dataset], session });
+    }
+
     await MongoDataset.findByIdAndUpdate(
       id,
       {
@@ -123,7 +163,7 @@ async function handler(
         ...(agentModel && { agentModel }),
         ...(vlmModel && { vlmModel }),
         ...(websiteConfig && { websiteConfig }),
-        ...(status && { status }),
+        ...(chunkSettings && { chunkSettings }),
         ...(intro !== undefined && { intro }),
         ...(externalReadUrl !== undefined && { externalReadUrl }),
         ...(!!apiServer?.baseUrl && { 'apiServer.baseUrl': apiServer.baseUrl }),
@@ -143,8 +183,7 @@ async function handler(
       { session }
     );
     await updateSyncSchedule({
-      teamId: dataset.teamId,
-      datasetId: dataset._id,
+      dataset,
       autoSync,
       session
     });
@@ -221,45 +260,54 @@ const updateTraining = async ({
 };
 
 const updateSyncSchedule = async ({
-  teamId,
-  datasetId,
+  dataset,
   autoSync,
   session
 }: {
-  teamId: string;
-  datasetId: string;
+  dataset: DatasetSchemaType;
   autoSync?: boolean;
   session: ClientSession;
 }) => {
   if (typeof autoSync !== 'boolean') return;
 
   // Update all collection nextSyncTime
-  if (autoSync) {
-    await MongoDatasetCollection.updateMany(
-      {
-        teamId,
-        datasetId,
-        type: { $in: [DatasetCollectionTypeEnum.apiFile, DatasetCollectionTypeEnum.link] }
-      },
-      {
-        $set: {
-          nextSyncTime: addDays(new Date(), 1)
-        }
-      },
-      { session }
-    );
+  if (dataset.type === DatasetTypeEnum.websiteDataset) {
+    if (autoSync) {
+      // upsert Job Scheduler
+      upsertWebsiteSyncJobScheduler({ datasetId: String(dataset._id) });
+    } else {
+      // remove Job Scheduler
+      removeWebsiteSyncJobScheduler(String(dataset._id));
+    }
   } else {
-    await MongoDatasetCollection.updateMany(
-      {
-        teamId,
-        datasetId
-      },
-      {
-        $unset: {
-          nextSyncTime: 1
-        }
-      },
-      { session }
-    );
+    // Other dataset, update the collection sync
+    if (autoSync) {
+      await MongoDatasetCollection.updateMany(
+        {
+          teamId: dataset.teamId,
+          datasetId: dataset._id,
+          type: { $in: [DatasetCollectionTypeEnum.apiFile, DatasetCollectionTypeEnum.link] }
+        },
+        {
+          $set: {
+            nextSyncTime: addDays(new Date(), 1)
+          }
+        },
+        { session }
+      );
+    } else {
+      await MongoDatasetCollection.updateMany(
+        {
+          teamId: dataset.teamId,
+          datasetId: dataset._id
+        },
+        {
+          $unset: {
+            nextSyncTime: 1
+          }
+        },
+        { session }
+      );
+    }
   }
 };

projects/app/src/pages/api/lafApi/[...path].ts

@@ -1,11 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { request } from 'https';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { path = [], ...query } = req.query as any;
 
     const queryStr = new URLSearchParams(query).toString();

projects/app/src/pages/api/proApi/[...path].ts

@@ -1,12 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { request } from 'http';
 import { FastGPTProUrl } from '@fastgpt/service/common/system/constants';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { path = [], ...query } = req.query as any;
     const requestPath = `/api/${path?.join('/')}?${new URLSearchParams(query).toString()}`;
 

projects/app/src/pages/api/support/user/account/updatePasswordByOld.ts

@@ -2,12 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { MongoUser } from '@fastgpt/service/support/user/schema';
-import { connectToDatabase } from '@/service/mongo';
+
 import { MongoTeamMember } from '@fastgpt/service/support/user/team/teamMemberSchema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
     const { oldPsw, newPsw } = req.body as { oldPsw: string; newPsw: string };
 
     if (!oldPsw || !newPsw) {

projects/app/src/pages/api/support/user/team/limit/datasetSizeLimit.ts

@@ -1,12 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 import { checkDatasetLimit } from '@fastgpt/service/support/permission/teamLimit';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
     const { size } = req.query as {
       size: string;
     };

projects/app/src/pages/api/support/user/team/limit/webSyncLimit.ts

@@ -1,13 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { checkWebSyncLimit } from '@fastgpt/service/support/user/utils';
 import { authCert } from '@fastgpt/service/support/permission/auth/common';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    await connectToDatabase();
-
     // 凭证校验
     const { teamId } = await authCert({ req, authToken: true });
 

projects/app/src/pages/api/system/img/[id].ts

@@ -1,12 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@fastgpt/service/common/response';
-import { connectToDatabase } from '@/service/mongo';
+
 import { readMongoImg } from '@fastgpt/service/common/file/image/controller';
 
 // get the models available to the system
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
-    await connectToDatabase();
     const { id } = req.query as { id: string };
 
     const { binary, mime } = await readMongoImg({ id });

projects/app/src/pages/api/v1/chat/completions.ts

@@ -59,7 +59,6 @@ import { getWorkflowResponseWrite } from '@fastgpt/service/core/workflow/dispatc
 import { WORKFLOW_MAX_RUN_TIMES } from '@fastgpt/service/core/workflow/constants';
 import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
 import { ExternalProviderType } from '@fastgpt/global/core/workflow/runtime/type';
-import { FlowNodeTypeEnum } from '@fastgpt/global/core/workflow/node/constant';
 
 type FastGptWebChatProps = {
   chatId?: string; // undefined: get histories from messages, '': new chat, 'xxxxx': get histories from db

projects/app/src/pages/api/v2/chat/completions.ts

@@ -0,0 +1,641 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { authApp } from '@fastgpt/service/support/permission/app/auth';
+import { authCert } from '@fastgpt/service/support/permission/auth/common';
+import { sseErrRes, jsonRes } from '@fastgpt/service/common/response';
+import { addLog } from '@fastgpt/service/common/system/log';
+import { ChatRoleEnum, ChatSourceEnum } from '@fastgpt/global/core/chat/constants';
+import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
+import { dispatchWorkFlow } from '@fastgpt/service/core/workflow/dispatch';
+import type { ChatCompletionCreateParams } from '@fastgpt/global/core/ai/type.d';
+import type { ChatCompletionMessageParam } from '@fastgpt/global/core/ai/type.d';
+import {
+  getWorkflowEntryNodeIds,
+  getMaxHistoryLimitFromNodes,
+  initWorkflowEdgeStatus,
+  storeNodes2RuntimeNodes,
+  textAdaptGptResponse,
+  getLastInteractiveValue
+} from '@fastgpt/global/core/workflow/runtime/utils';
+import { GPTMessages2Chats, chatValue2RuntimePrompt } from '@fastgpt/global/core/chat/adapt';
+import { getChatItems } from '@fastgpt/service/core/chat/controller';
+import { saveChat, updateInteractiveChat } from '@fastgpt/service/core/chat/saveChat';
+import { responseWrite } from '@fastgpt/service/common/response';
+import { createChatUsage } from '@fastgpt/service/support/wallet/usage/controller';
+import { authOutLinkChatStart } from '@/service/support/permission/auth/outLink';
+import { pushResult2Remote, addOutLinkUsage } from '@fastgpt/service/support/outLink/tools';
+import requestIp from 'request-ip';
+import { getUsageSourceByAuthType } from '@fastgpt/global/support/wallet/usage/tools';
+import { authTeamSpaceToken } from '@/service/support/permission/auth/team';
+import {
+  concatHistories,
+  filterPublicNodeResponseData,
+  getChatTitleFromChatMessage,
+  removeEmptyUserInput
+} from '@fastgpt/global/core/chat/utils';
+import { updateApiKeyUsage } from '@fastgpt/service/support/openapi/tools';
+import { getUserChatInfoAndAuthTeamPoints } from '@fastgpt/service/support/permission/auth/team';
+import { AuthUserTypeEnum } from '@fastgpt/global/support/permission/constant';
+import { MongoApp } from '@fastgpt/service/core/app/schema';
+import { AppSchema } from '@fastgpt/global/core/app/type';
+import { AuthOutLinkChatProps } from '@fastgpt/global/support/outLink/api';
+import { MongoChat } from '@fastgpt/service/core/chat/chatSchema';
+import { ChatErrEnum } from '@fastgpt/global/common/error/code/chat';
+import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
+import { AIChatItemType, UserChatItemType } from '@fastgpt/global/core/chat/type';
+import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
+
+import { NextAPI } from '@/service/middleware/entry';
+import { getAppLatestVersion } from '@fastgpt/service/core/app/version/controller';
+import { ReadPermissionVal } from '@fastgpt/global/support/permission/constant';
+import { AppTypeEnum } from '@fastgpt/global/core/app/constants';
+import {
+  getPluginRunUserQuery,
+  updatePluginInputByVariables
+} from '@fastgpt/global/core/workflow/utils';
+import { getNanoid } from '@fastgpt/global/common/string/tools';
+import { getSystemTime } from '@fastgpt/global/common/time/timezone';
+import { rewriteNodeOutputByHistories } from '@fastgpt/global/core/workflow/runtime/utils';
+import { getWorkflowResponseWrite } from '@fastgpt/service/core/workflow/dispatch/utils';
+import { WORKFLOW_MAX_RUN_TIMES } from '@fastgpt/service/core/workflow/constants';
+import { getPluginInputsFromStoreNodes } from '@fastgpt/global/core/app/plugin/utils';
+import { ExternalProviderType } from '@fastgpt/global/core/workflow/runtime/type';
+
+type FastGptWebChatProps = {
+  chatId?: string; // undefined: get histories from messages, '': new chat, 'xxxxx': get histories from db
+  appId?: string;
+  customUid?: string; // non-undefined: will be the priority provider for the logger.
+  metadata?: Record<string, any>;
+};
+
+export type Props = ChatCompletionCreateParams &
+  FastGptWebChatProps &
+  OutLinkChatAuthProps & {
+    messages: ChatCompletionMessageParam[];
+    responseChatItemId?: string;
+    stream?: boolean;
+    detail?: boolean;
+    variables: Record<string, any>; // Global variables or plugin inputs
+  };
+
+type AuthResponseType = {
+  teamId: string;
+  tmbId: string;
+  timezone: string;
+  externalProvider: ExternalProviderType;
+  app: AppSchema;
+  responseDetail?: boolean;
+  showNodeStatus?: boolean;
+  authType: `${AuthUserTypeEnum}`;
+  apikey?: string;
+  responseAllData: boolean;
+  outLinkUserId?: string;
+  sourceName?: string;
+};
+
+async function handler(req: NextApiRequest, res: NextApiResponse) {
+  res.on('close', () => {
+    res.end();
+  });
+  res.on('error', () => {
+    console.log('error: ', 'request error');
+    res.end();
+  });
+
+  let {
+    chatId,
+    appId,
+    customUid,
+    // share chat
+    shareId,
+    outLinkUid,
+    // team chat
+    teamId: spaceTeamId,
+    teamToken,
+
+    stream = false,
+    detail = false,
+    messages = [],
+    variables = {},
+    responseChatItemId = getNanoid(),
+    metadata
+  } = req.body as Props;
+
+  const originIp = requestIp.getClientIp(req);
+
+  const startTime = Date.now();
+
+  try {
+    if (!Array.isArray(messages)) {
+      throw new Error('messages is not array');
+    }
+
+    /*
+      Web params: chatId + [Human]
+      API params: chatId + [Human]
+      API params: [histories, Human]
+    */
+    const chatMessages = GPTMessages2Chats(messages);
+
+    // Computed start hook params
+    const startHookText = (() => {
+      // Chat
+      const userQuestion = chatMessages[chatMessages.length - 1] as UserChatItemType | undefined;
+      if (userQuestion) return chatValue2RuntimePrompt(userQuestion.value).text;
+
+      // plugin
+      return JSON.stringify(variables);
+    })();
+
+    /*
+      1. auth app permission
+      2. auth balance
+      3. get app
+      4. parse outLink token
+    */
+    const {
+      teamId,
+      tmbId,
+      timezone,
+      externalProvider,
+      app,
+      responseDetail,
+      authType,
+      sourceName,
+      apikey,
+      responseAllData,
+      outLinkUserId = customUid,
+      showNodeStatus
+    } = await (async () => {
+      // share chat
+      if (shareId && outLinkUid) {
+        return authShareChat({
+          shareId,
+          outLinkUid,
+          chatId,
+          ip: originIp,
+          question: startHookText
+        });
+      }
+      // team space chat
+      if (spaceTeamId && appId && teamToken) {
+        return authTeamSpaceChat({
+          teamId: spaceTeamId,
+          teamToken,
+          appId,
+          chatId
+        });
+      }
+
+      /* parse req: api or token */
+      return authHeaderRequest({
+        req,
+        appId,
+        chatId
+      });
+    })();
+    const isPlugin = app.type === AppTypeEnum.plugin;
+
+    // Check message type
+    if (isPlugin) {
+      detail = true;
+    } else {
+      if (messages.length === 0) {
+        throw new Error('messages is empty');
+      }
+    }
+
+    // Get obj=Human history
+    const userQuestion: UserChatItemType = (() => {
+      if (isPlugin) {
+        return getPluginRunUserQuery({
+          pluginInputs: getPluginInputsFromStoreNodes(app.modules),
+          variables,
+          files: variables.files
+        });
+      }
+
+      const latestHumanChat = chatMessages.pop() as UserChatItemType | undefined;
+      if (!latestHumanChat) {
+        throw new Error('User question is empty');
+      }
+      return latestHumanChat;
+    })();
+
+    // Get and concat history;
+    const limit = getMaxHistoryLimitFromNodes(app.modules);
+    const [{ histories }, { nodes, edges, chatConfig }, chatDetail] = await Promise.all([
+      getChatItems({
+        appId: app._id,
+        chatId,
+        offset: 0,
+        limit,
+        field: `dataId obj value nodeOutputs`
+      }),
+      getAppLatestVersion(app._id, app),
+      MongoChat.findOne({ appId: app._id, chatId }, 'source variableList variables')
+    ]);
+
+    // Get store variables(Api variable precedence)
+    if (chatDetail?.variables) {
+      variables = {
+        ...chatDetail.variables,
+        ...variables
+      };
+    }
+
+    // Get chat histories
+    const newHistories = concatHistories(histories, chatMessages);
+
+    // Get runtimeNodes
+    let runtimeNodes = storeNodes2RuntimeNodes(nodes, getWorkflowEntryNodeIds(nodes, newHistories));
+    if (isPlugin) {
+      // Assign values to runtimeNodes using variables
+      runtimeNodes = updatePluginInputByVariables(runtimeNodes, variables);
+      // Plugin runtime does not need global variables(It has been injected into the pluginInputNode)
+      variables = {};
+    }
+    runtimeNodes = rewriteNodeOutputByHistories(newHistories, runtimeNodes);
+
+    const workflowResponseWrite = getWorkflowResponseWrite({
+      res,
+      detail,
+      streamResponse: stream,
+      id: chatId,
+      showNodeStatus
+    });
+
+    /* start flow controller */
+    const { flowResponses, flowUsages, assistantResponses, newVariables } = await (async () => {
+      if (app.version === 'v2') {
+        return dispatchWorkFlow({
+          res,
+          requestOrigin: req.headers.origin,
+          mode: 'chat',
+          timezone,
+          externalProvider,
+
+          runningAppInfo: {
+            id: String(app._id),
+            teamId: String(app.teamId),
+            tmbId: String(app.tmbId)
+          },
+          runningUserInfo: {
+            teamId,
+            tmbId
+          },
+          uid: String(outLinkUserId || tmbId),
+
+          chatId,
+          responseChatItemId,
+          runtimeNodes,
+          runtimeEdges: initWorkflowEdgeStatus(edges, newHistories),
+          variables,
+          query: removeEmptyUserInput(userQuestion.value),
+          chatConfig,
+          histories: newHistories,
+          stream,
+          maxRunTimes: WORKFLOW_MAX_RUN_TIMES,
+          workflowStreamResponse: workflowResponseWrite,
+          version: 'v2'
+        });
+      }
+      return Promise.reject('您的工作流版本过低，请重新发布一次');
+    })();
+
+    // save chat
+    const isOwnerUse = !shareId && !spaceTeamId && String(tmbId) === String(app.tmbId);
+    const source = (() => {
+      if (shareId) {
+        return ChatSourceEnum.share;
+      }
+      if (authType === 'apikey') {
+        return ChatSourceEnum.api;
+      }
+      if (spaceTeamId) {
+        return ChatSourceEnum.team;
+      }
+      return ChatSourceEnum.online;
+    })();
+
+    const isInteractiveRequest = !!getLastInteractiveValue(histories);
+    const { text: userInteractiveVal } = chatValue2RuntimePrompt(userQuestion.value);
+
+    const newTitle = isPlugin
+      ? variables.cTime ?? getSystemTime(timezone)
+      : getChatTitleFromChatMessage(userQuestion);
+
+    const aiResponse: AIChatItemType & { dataId?: string } = {
+      dataId: responseChatItemId,
+      obj: ChatRoleEnum.AI,
+      value: assistantResponses,
+      [DispatchNodeResponseKeyEnum.nodeResponse]: flowResponses
+    };
+
+    const saveChatId = chatId || getNanoid(24);
+    if (isInteractiveRequest) {
+      await updateInteractiveChat({
+        chatId: saveChatId,
+        appId: app._id,
+        userInteractiveVal,
+        aiResponse,
+        newVariables
+      });
+    } else {
+      await saveChat({
+        chatId: saveChatId,
+        appId: app._id,
+        teamId,
+        tmbId: tmbId,
+        nodes,
+        appChatConfig: chatConfig,
+        variables: newVariables,
+        isUpdateUseTime: isOwnerUse && source === ChatSourceEnum.online, // owner update use time
+        newTitle,
+        shareId,
+        outLinkUid: outLinkUserId,
+        source,
+        sourceName: sourceName || '',
+        content: [userQuestion, aiResponse],
+        metadata: {
+          originIp,
+          ...metadata
+        }
+      });
+    }
+
+    addLog.info(`completions running time: ${(Date.now() - startTime) / 1000}s`);
+
+    /* select fe response field */
+    const feResponseData = responseAllData
+      ? flowResponses
+      : filterPublicNodeResponseData({ flowResponses, responseDetail });
+
+    if (stream) {
+      workflowResponseWrite({
+        event: SseResponseEventEnum.answer,
+        data: textAdaptGptResponse({
+          text: null,
+          finish_reason: 'stop'
+        })
+      });
+      responseWrite({
+        res,
+        event: detail ? SseResponseEventEnum.answer : undefined,
+        data: '[DONE]'
+      });
+
+      res.end();
+    } else {
+      const responseContent = (() => {
+        if (assistantResponses.length === 0) return '';
+        if (assistantResponses.length === 1 && assistantResponses[0].text?.content)
+          return assistantResponses[0].text?.content;
+
+        if (!detail) {
+          return assistantResponses
+            .map((item) => item?.text?.content)
+            .filter(Boolean)
+            .join('\n');
+        }
+
+        return assistantResponses;
+      })();
+      const error = flowResponses[flowResponses.length - 1]?.error;
+
+      res.json({
+        ...(detail ? { responseData: feResponseData, newVariables } : {}),
+        error,
+        id: chatId || '',
+        model: '',
+        usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 1 },
+        choices: [
+          {
+            message: { role: 'assistant', content: responseContent },
+            finish_reason: 'stop',
+            index: 0
+          }
+        ]
+      });
+    }
+
+    // add record
+    const { totalPoints } = createChatUsage({
+      appName: app.name,
+      appId: app._id,
+      teamId,
+      tmbId: tmbId,
+      source: getUsageSourceByAuthType({ shareId, authType }),
+      flowUsages
+    });
+
+    if (shareId) {
+      pushResult2Remote({ outLinkUid, shareId, appName: app.name, flowResponses });
+      addOutLinkUsage({
+        shareId,
+        totalPoints
+      });
+    }
+    if (apikey) {
+      updateApiKeyUsage({
+        apikey,
+        totalPoints
+      });
+    }
+  } catch (err) {
+    if (stream) {
+      sseErrRes(res, err);
+      res.end();
+    } else {
+      jsonRes(res, {
+        code: 500,
+        error: err
+      });
+    }
+  }
+}
+export default NextAPI(handler);
+
+const authShareChat = async ({
+  chatId,
+  ...data
+}: AuthOutLinkChatProps & {
+  shareId: string;
+  chatId?: string;
+}): Promise<AuthResponseType> => {
+  const {
+    teamId,
+    tmbId,
+    timezone,
+    externalProvider,
+    appId,
+    authType,
+    responseDetail,
+    showNodeStatus,
+    uid,
+    sourceName
+  } = await authOutLinkChatStart(data);
+  const app = await MongoApp.findById(appId).lean();
+
+  if (!app) {
+    return Promise.reject('app is empty');
+  }
+
+  // get chat
+  const chat = await MongoChat.findOne({ appId, chatId }).lean();
+  if (chat && (chat.shareId !== data.shareId || chat.outLinkUid !== uid)) {
+    return Promise.reject(ChatErrEnum.unAuthChat);
+  }
+
+  return {
+    sourceName,
+    teamId,
+    tmbId,
+    app,
+    timezone,
+    externalProvider,
+    apikey: '',
+    authType,
+    responseAllData: false,
+    responseDetail,
+    outLinkUserId: uid,
+    showNodeStatus
+  };
+};
+const authTeamSpaceChat = async ({
+  appId,
+  teamId,
+  teamToken,
+  chatId
+}: {
+  appId: string;
+  teamId: string;
+  teamToken: string;
+  chatId?: string;
+}): Promise<AuthResponseType> => {
+  const { uid } = await authTeamSpaceToken({
+    teamId,
+    teamToken
+  });
+
+  const app = await MongoApp.findById(appId).lean();
+  if (!app) {
+    return Promise.reject('app is empty');
+  }
+
+  const [chat, { timezone, externalProvider }] = await Promise.all([
+    MongoChat.findOne({ appId, chatId }).lean(),
+    getUserChatInfoAndAuthTeamPoints(app.tmbId)
+  ]);
+
+  if (chat && (String(chat.teamId) !== teamId || chat.outLinkUid !== uid)) {
+    return Promise.reject(ChatErrEnum.unAuthChat);
+  }
+
+  return {
+    teamId,
+    tmbId: app.tmbId,
+    app,
+    timezone,
+    externalProvider,
+    authType: AuthUserTypeEnum.outLink,
+    apikey: '',
+    responseAllData: false,
+    responseDetail: true,
+    outLinkUserId: uid
+  };
+};
+const authHeaderRequest = async ({
+  req,
+  appId,
+  chatId
+}: {
+  req: NextApiRequest;
+  appId?: string;
+  chatId?: string;
+}): Promise<AuthResponseType> => {
+  const {
+    appId: apiKeyAppId,
+    teamId,
+    tmbId,
+    authType,
+    sourceName,
+    apikey
+  } = await authCert({
+    req,
+    authToken: true,
+    authApiKey: true
+  });
+
+  const { app } = await (async () => {
+    if (authType === AuthUserTypeEnum.apikey) {
+      const currentAppId = apiKeyAppId || appId;
+      if (!currentAppId) {
+        return Promise.reject(
+          'Key is error. You need to use the app key rather than the account key.'
+        );
+      }
+      const app = await MongoApp.findById(currentAppId);
+
+      if (!app) {
+        return Promise.reject('app is empty');
+      }
+
+      appId = String(app._id);
+
+      return {
+        app
+      };
+    } else {
+      // token_auth
+      if (!appId) {
+        return Promise.reject('appId is empty');
+      }
+      const { app } = await authApp({
+        req,
+        authToken: true,
+        appId,
+        per: ReadPermissionVal
+      });
+
+      return {
+        app
+      };
+    }
+  })();
+
+  const [{ timezone, externalProvider }, chat] = await Promise.all([
+    getUserChatInfoAndAuthTeamPoints(tmbId),
+    MongoChat.findOne({ appId, chatId }).lean()
+  ]);
+
+  if (
+    chat &&
+    (String(chat.teamId) !== teamId ||
+      // There's no need to distinguish who created it if it's apiKey auth
+      (authType === AuthUserTypeEnum.token && String(chat.tmbId) !== tmbId))
+  ) {
+    return Promise.reject(ChatErrEnum.unAuthChat);
+  }
+
+  return {
+    teamId,
+    tmbId,
+    timezone,
+    externalProvider,
+    app,
+    apikey,
+    authType,
+    sourceName,
+    responseAllData: true,
+    responseDetail: true
+  };
+};
+
+export const config = {
+  api: {
+    bodyParser: {
+      sizeLimit: '20mb'
+    },
+    responseLimit: '20mb'
+  }
+};

projects/app/src/pages/chat/index.tsx

@@ -115,7 +115,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
     }: StartChatFnProps) => {
       // Just send a user prompt
       const histories = messages.slice(-1);
-      const { responseText, responseData } = await streamFetch({
+      const { responseText } = await streamFetch({
         data: {
           messages: histories,
           variables,
@@ -137,7 +137,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
         title: newTitle
       }));
 
-      return { responseText, responseData, isNewChat: forbidLoadChat.current };
+      return { responseText, isNewChat: forbidLoadChat.current };
     },
     [appId, chatId, onUpdateHistoryTitle, setChatBoxData, forbidLoadChat]
   );
@@ -174,13 +174,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
       )}
 
       {(!quoteData || isPc) && (
-        <PageContainer
-          isLoading={loading}
-          flex={'1 0 0'}
-          w={0}
-          p={[0, '16px']}
-          position={'relative'}
-        >
+        <PageContainer flex={'1 0 0'} w={0} p={[0, '16px']} position={'relative'}>
           <Flex h={'100%'} flexDirection={['column', 'row']}>
             {/* pc always show history. */}
             {RenderHistorySlider}

projects/app/src/pages/chat/share.tsx

@@ -17,7 +17,7 @@ import { getInitOutLinkChatInfo } from '@/web/core/chat/api';
 import { getChatTitleFromChatMessage } from '@fastgpt/global/core/chat/utils';
 import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
 import { addLog } from '@fastgpt/service/common/system/log';
-import { connectToDatabase } from '@/service/mongo';
+
 import NextHead from '@/components/common/NextHead';
 import { useContextSelector } from 'use-context-selector';
 import ChatContextProvider, { ChatContext } from '@/web/core/chat/context/chatContext';
@@ -151,7 +151,7 @@ const OutLink = (props: Props) => {
         '*'
       );
 
-      const { responseText, responseData } = await streamFetch({
+      const { responseText } = await streamFetch({
         data: {
           messages: histories,
           variables: {
@@ -192,7 +192,7 @@ const OutLink = (props: Props) => {
         '*'
       );
 
-      return { responseText, responseData, isNewChat: forbidLoadChat.current };
+      return { responseText, isNewChat: forbidLoadChat.current };
     },
     [
       chatId,
@@ -251,7 +251,7 @@ const OutLink = (props: Props) => {
         {...(isEmbed ? { p: '0 !important', borderRadius: '0', boxShadow: 'none' } : { p: [0, 5] })}
       >
         {(!quoteData || isPc) && (
-          <PageContainer flex={'1 0 0'} w={0} isLoading={loading} p={'0 !important'}>
+          <PageContainer flex={'1 0 0'} w={0} p={'0 !important'}>
             <Flex h={'100%'} flexDirection={['column', 'row']}>
               {RenderHistoryList}
 
@@ -391,7 +391,6 @@ export async function getServerSideProps(context: any) {
 
   const app = await (async () => {
     try {
-      await connectToDatabase();
       return MongoOutLink.findOne(
         {
           shareId

projects/app/src/pages/chat/team.tsx

@@ -112,7 +112,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
       // Just send a user prompt
       const histories = messages.slice(-1);
 
-      const { responseText, responseData } = await streamFetch({
+      const { responseText } = await streamFetch({
         data: {
           messages: histories,
           variables: {
@@ -144,7 +144,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
         title: newTitle
       }));
 
-      return { responseText, responseData, isNewChat: forbidLoadChat.current };
+      return { responseText, isNewChat: forbidLoadChat.current };
     },
     [
       chatId,
@@ -192,13 +192,7 @@ const Chat = ({ myApps }: { myApps: AppListItemType[] }) => {
       )}
 
       {(!quoteData || isPc) && (
-        <PageContainer
-          isLoading={loading}
-          flex={'1 0 0'}
-          w={0}
-          p={[0, '16px']}
-          position={'relative'}
-        >
+        <PageContainer flex={'1 0 0'} w={0} p={[0, '16px']} position={'relative'}>
           <Flex h={'100%'} flexDirection={['column', 'row']} bg={'white'}>
             {RenderHistoryList}
             {/* chat container */}

projects/app/src/service/common/system/cron.ts

@@ -28,10 +28,11 @@ const setClearTmpUploadFilesCron = () => {
 };
 
 const clearInvalidDataCron = () => {
+  // Clear files
   setCron('0 */1 * * *', async () => {
     if (
       await checkTimerLock({
-        timerId: TimerIdEnum.checkInValidDatasetFiles,
+        timerId: TimerIdEnum.checkExpiredFiles,
         lockMinuted: 59
       })
     ) {

projects/app/src/service/common/system/cronTask.ts

@@ -71,8 +71,10 @@ export async function checkInvalidDatasetFiles(start: Date, end: Date) {
 export const removeExpiredChatFiles = async () => {
   let deleteFileAmount = 0;
   const collection = getGFSCollection(BucketNameEnum.chat);
+
+  const expireTime = Number(process.env.CHAT_FILE_EXPIRE_TIME || 7);
   const where = {
-    uploadDate: { $lte: addDays(new Date(), -7) }
+    uploadDate: { $lte: addDays(new Date(), -expireTime) }
   };
 
   // get all file _id

projects/app/src/service/events/generateQA.ts

@@ -26,6 +26,7 @@ import {
   chunkAutoChunkSize,
   getLLMMaxChunkSize
 } from '@fastgpt/global/core/dataset/training/utils';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 
 const reduceQueue = () => {
   global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0;
@@ -50,7 +51,7 @@ export async function generateQA(): Promise<any> {
       const data = await MongoDatasetTraining.findOneAndUpdate(
         {
           mode: TrainingModeEnum.qa,
-          retryCount: { $gte: 0 },
+          retryCount: { $gt: 0 },
           lockTime: { $lte: addMinutes(new Date(), -10) }
         },
         {
@@ -176,7 +177,16 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`;
     generateQA();
   } catch (err: any) {
     addLog.error(`[QA Queue] Error`, err);
-    reduceQueue();
+    await MongoDatasetTraining.updateOne(
+      {
+        teamId: data.teamId,
+        datasetId: data.datasetId,
+        _id: data._id
+      },
+      {
+        errorMsg: getErrText(err, 'unknown error')
+      }
+    );
 
     setTimeout(() => {
       generateQA();

projects/app/src/service/events/generateVector.ts

@@ -14,6 +14,7 @@ import { getEmbeddingModel } from '@fastgpt/service/core/ai/model';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
 import { DatasetTrainingSchemaType } from '@fastgpt/global/core/dataset/type';
 import { Document } from '@fastgpt/service/common/mongo';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 
 const reduceQueue = () => {
   global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0;
@@ -48,7 +49,7 @@ export async function generateVector(): Promise<any> {
       const data = await MongoDatasetTraining.findOneAndUpdate(
         {
           mode: TrainingModeEnum.chunk,
-          retryCount: { $gte: 0 },
+          retryCount: { $gt: 0 },
           lockTime: { $lte: addMinutes(new Date(), -3) }
         },
         {
@@ -117,6 +118,16 @@ export async function generateVector(): Promise<any> {
     return reduceQueueAndReturn();
   } catch (err: any) {
     addLog.error(`[Vector Queue] Error`, err);
+    await MongoDatasetTraining.updateOne(
+      {
+        teamId: data.teamId,
+        datasetId: data.datasetId,
+        _id: data._id
+      },
+      {
+        errorMsg: getErrText(err, 'unknown error')
+      }
+    );
     return reduceQueueAndReturn(1000);
   }
 }

projects/app/src/service/mongo.ts

@@ -1,19 +1,9 @@
-import { PRICE_SCALE } from '@fastgpt/global/support/wallet/constants';
 import { MongoUser } from '@fastgpt/service/support/user/schema';
-import { connectMongo } from '@fastgpt/service/common/mongo/init';
 import { hashStr } from '@fastgpt/global/common/string/tools';
 import { createDefaultTeam } from '@fastgpt/service/support/user/team/controller';
 import { exit } from 'process';
 import { mongoSessionRun } from '@fastgpt/service/common/mongo/sessionRun';
 
-/**
- * This function is equivalent to the entry to the service
- * connect MongoDB and init data
- */
-export function connectToDatabase() {
-  return connectMongo();
-}
-
 export async function initRootUser(retry = 3): Promise<any> {
   try {
     const rootUser = await MongoUser.findOne({

projects/app/src/web/common/api/fetch.ts

@@ -1,8 +1,6 @@
 import { SseResponseEventEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import { getErrText } from '@fastgpt/global/common/error/utils';
-import type { ChatHistoryItemResType } from '@fastgpt/global/core/chat/type.d';
 import type { StartChatFnProps } from '@/components/core/chat/ChatContainer/type';
-import { DispatchNodeResponseKeyEnum } from '@fastgpt/global/core/workflow/runtime/constants';
 import {
   // refer to https://github.com/ChatGPTNextWeb/ChatGPT-Next-Web
   EventStreamContentType,
@@ -21,7 +19,6 @@ type StreamFetchProps = {
 };
 export type StreamResponseType = {
   responseText: string;
-  [DispatchNodeResponseKeyEnum.nodeResponse]: ChatHistoryItemResType[];
 };
 type ResponseQueueItemType =
   | {
@@ -40,7 +37,7 @@ type ResponseQueueItemType =
 class FatalError extends Error {}
 
 export const streamFetch = ({
-  url = '/api/v1/chat/completions',
+  url = '/api/v2/chat/completions',
   data,
   onMessage,
   abortCtrl
@@ -55,7 +52,6 @@ export const streamFetch = ({
     let responseText = '';
     let responseQueue: ResponseQueueItemType[] = [];
     let errMsg: string | undefined;
-    let responseData: ChatHistoryItemResType[] = [];
     let finished = false;
 
     const finish = () => {
@@ -63,8 +59,7 @@ export const streamFetch = ({
         return failedFinish();
       }
       return resolve({
-        responseText,
-        responseData
+        responseText
       });
     };
     const failedFinish = (err?: any) => {
@@ -168,7 +163,7 @@ export const streamFetch = ({
             }
           }
         },
-        onmessage({ event, data }) {
+        onmessage: ({ event, data }) => {
           if (data === '[DONE]') {
             return;
           }
@@ -178,9 +173,12 @@ export const streamFetch = ({
             try {
               return JSON.parse(data);
             } catch (error) {
-              return {};
+              return;
             }
           })();
+
+          if (typeof parseJson !== 'object') return;
+
           // console.log(parseJson, event);
           if (event === SseResponseEventEnum.answer) {
             const reasoningText = parseJson.choices?.[0]?.delta?.reasoning_content || '';
@@ -222,8 +220,11 @@ export const streamFetch = ({
               event,
               ...parseJson
             });
-          } else if (event === SseResponseEventEnum.flowResponses && Array.isArray(parseJson)) {
-            responseData = parseJson;
+          } else if (event === SseResponseEventEnum.flowNodeResponse) {
+            onMessage({
+              event,
+              nodeResponse: parseJson
+            });
           } else if (event === SseResponseEventEnum.updateVariables) {
             onMessage({
               event,

projects/app/src/web/common/hooks/useSpeech.ts

@@ -7,16 +7,21 @@ import { OutLinkChatAuthProps } from '@fastgpt/global/support/permission/chat';
 
 export const useSpeech = (props?: OutLinkChatAuthProps & { appId?: string }) => {
   const { t } = useTranslation();
-  const mediaRecorder = useRef<MediaRecorder>();
-  const [mediaStream, setMediaStream] = useState<MediaStream>();
   const { toast } = useToast();
+
   const [isSpeaking, setIsSpeaking] = useState(false);
   const [isTransCription, setIsTransCription] = useState(false);
-  const [audioSecond, setAudioSecond] = useState(0);
-  const intervalRef = useRef<any>();
-  const startTimestamp = useRef(0);
-  const cancelWhisperSignal = useRef(false);
 
+  const mediaRecorder = useRef<MediaRecorder>();
+  const [mediaStream, setMediaStream] = useState<MediaStream>();
+
+  const timeIntervalRef = useRef<any>();
+  const cancelWhisperSignal = useRef(false);
+  const stopCalledRef = useRef(false);
+
+  const startTimestamp = useRef(0);
+
+  const [audioSecond, setAudioSecond] = useState(0);
   const speakingTimeString = useMemo(() => {
     const minutes: number = Math.floor(audioSecond / 60);
     const remainingSeconds: number = Math.floor(audioSecond % 60);
@@ -25,17 +30,16 @@ export const useSpeech = (props?: OutLinkChatAuthProps & { appId?: string }) =>
     return `${formattedMinutes}:${formattedSeconds}`;
   }, [audioSecond]);
 
-  const renderAudioGraph = useCallback((analyser: AnalyserNode, canvas: HTMLCanvasElement) => {
+  const renderAudioGraphPc = useCallback((analyser: AnalyserNode, canvas: HTMLCanvasElement) => {
     const bufferLength = analyser.frequencyBinCount;
-    const backgroundColor = 'white';
     const dataArray = new Uint8Array(bufferLength);
     analyser.getByteTimeDomainData(dataArray);
     const canvasCtx = canvas?.getContext('2d');
-    const width = 300;
-    const height = 200;
+    const width = canvas.width;
+    const height = canvas.height;
     if (!canvasCtx) return;
     canvasCtx.clearRect(0, 0, width, height);
-    canvasCtx.fillStyle = backgroundColor;
+    canvasCtx.fillStyle = 'white';
     canvasCtx.fillRect(0, 0, width, height);
     const barWidth = (width / bufferLength) * 2.5;
     let x = 0;
@@ -49,127 +53,212 @@ export const useSpeech = (props?: OutLinkChatAuthProps & { appId?: string }) =>
       x += barWidth + 1;
     }
   }, []);
+  const renderAudioGraphMobile = useCallback(
+    (analyser: AnalyserNode, canvas: HTMLCanvasElement) => {
+      const canvasCtx = canvas?.getContext('2d');
+      if (!canvasCtx) return;
 
-  const startSpeak = async (onFinish: (text: string) => void) => {
-    if (!navigator?.mediaDevices?.getUserMedia) {
-      return toast({
-        status: 'warning',
-        title: t('common:common.speech.not support')
-      });
-    }
-    try {
+      const bufferLength = analyser.frequencyBinCount;
+      const dataArray = new Uint8Array(bufferLength);
+      analyser.getByteTimeDomainData(dataArray);
+
+      const width = canvas.width;
+      const height = canvas.height;
+      canvasCtx.clearRect(0, 0, width, height);
+
+      // Set transparent background
+      canvasCtx.fillStyle = 'rgba(255, 255, 255, 0)';
+      canvasCtx.fillRect(0, 0, width, height);
+
+      const centerY = height / 2;
+      const barWidth = (width / bufferLength) * 15;
+      const gap = 2; // 添加间隙
+      let x = width * 0.1;
+
+      let sum = 0;
+      let maxDiff = 0;
+
+      for (let i = 0; i < bufferLength; i++) {
+        sum += dataArray[i];
+        maxDiff = Math.max(maxDiff, Math.abs(dataArray[i] - 128));
+      }
+      const average = sum / bufferLength;
+
+      // draw initial rectangle waveform
+      canvasCtx.beginPath();
+      canvasCtx.fillStyle = '#FFFFFF';
+
+      const initialHeight = height * 0.1;
+      for (let i = 0; i < width * 0.8; i += barWidth + gap) {
+        canvasCtx.fillRect(i + width * 0.1, centerY - initialHeight, barWidth, initialHeight);
+        canvasCtx.fillRect(i + width * 0.1, centerY, barWidth, initialHeight);
+      }
+
+      // draw dynamic waveform
+      canvasCtx.beginPath();
+      for (let i = 0; i < bufferLength; i += 4) {
+        const value = dataArray[i];
+        const normalizedValue = (value - average) / 128;
+        const amplification = 2.5;
+        const barHeight = normalizedValue * height * 0.4 * amplification;
+
+        canvasCtx.fillStyle = '#FFFFFF';
+
+        canvasCtx.fillRect(x, centerY - Math.abs(barHeight), barWidth, Math.abs(barHeight));
+        canvasCtx.fillRect(x, centerY, barWidth, Math.abs(barHeight));
+
+        x += barWidth + gap; // 增加间隔
+
+        if (x > width * 0.9) break;
+      }
+    },
+    []
+  );
+
+  const startSpeak = useCallback(
+    async (onFinish: (text: string) => void) => {
+      if (!navigator?.mediaDevices?.getUserMedia) {
+        return toast({
+          status: 'warning',
+          title: t('common:common.speech.not support')
+        });
+      }
+
+      // Init status
+      if (timeIntervalRef.current) {
+        clearInterval(timeIntervalRef.current);
+      }
       cancelWhisperSignal.current = false;
+      stopCalledRef.current = false;
 
-      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
-      setMediaStream(stream);
-
-      mediaRecorder.current = new MediaRecorder(stream);
-      const chunks: Blob[] = [];
       setIsSpeaking(true);
+      setAudioSecond(0);
 
-      mediaRecorder.current.onstart = () => {
-        startTimestamp.current = Date.now();
-        setAudioSecond(0);
-        intervalRef.current = setInterval(() => {
-          const currentTimestamp = Date.now();
-          const duration = (currentTimestamp - startTimestamp.current) / 1000;
-          setAudioSecond(duration);
-        }, 1000);
-      };
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+        setMediaStream(stream);
 
-      mediaRecorder.current.ondataavailable = (e) => {
-        chunks.push(e.data);
-      };
+        mediaRecorder.current = new MediaRecorder(stream);
+        const chunks: Blob[] = [];
 
-      mediaRecorder.current.onstop = async () => {
-        if (!cancelWhisperSignal.current) {
-          const formData = new FormData();
-          const { options, filename } = (() => {
-            if (MediaRecorder.isTypeSupported('video/webm; codecs=vp9')) {
-              return {
-                options: { mimeType: 'video/webm; codecs=vp9' },
-                filename: 'recording.mp3'
-              };
-            }
-            if (MediaRecorder.isTypeSupported('video/webm')) {
+        mediaRecorder.current.onstart = () => {
+          startTimestamp.current = Date.now();
+          timeIntervalRef.current = setInterval(() => {
+            const currentTimestamp = Date.now();
+            const duration = (currentTimestamp - startTimestamp.current) / 1000;
+            setAudioSecond(duration);
+          }, 1000);
+        };
+        mediaRecorder.current.ondataavailable = (e) => {
+          chunks.push(e.data);
+        };
+        mediaRecorder.current.onstop = async () => {
+          // close media stream
+          stream.getTracks().forEach((track) => track.stop());
+          setIsSpeaking(false);
+
+          if (timeIntervalRef.current) {
+            clearInterval(timeIntervalRef.current);
+          }
+
+          if (!cancelWhisperSignal.current) {
+            const formData = new FormData();
+            const { options, filename } = (() => {
+              if (MediaRecorder.isTypeSupported('video/webm; codecs=vp9')) {
+                return {
+                  options: { mimeType: 'video/webm; codecs=vp9' },
+                  filename: 'recording.mp3'
+                };
+              }
+              if (MediaRecorder.isTypeSupported('video/webm')) {
+                return {
+                  options: { type: 'video/webm' },
+                  filename: 'recording.mp3'
+                };
+              }
+              if (MediaRecorder.isTypeSupported('video/mp4')) {
+                return {
+                  options: { mimeType: 'video/mp4', videoBitsPerSecond: 100000 },
+                  filename: 'recording.mp4'
+                };
+              }
               return {
                 options: { type: 'video/webm' },
                 filename: 'recording.mp3'
               };
-            }
-            if (MediaRecorder.isTypeSupported('video/mp4')) {
-              return {
-                options: { mimeType: 'video/mp4', videoBitsPerSecond: 100000 },
-                filename: 'recording.mp4'
-              };
-            }
-            return {
-              options: { type: 'video/webm' },
-              filename: 'recording.mp3'
-            };
-          })();
+            })();
 
-          const blob = new Blob(chunks, options);
-          const duration = Math.round((Date.now() - startTimestamp.current) / 1000);
-          formData.append('file', blob, filename);
-          formData.append(
-            'data',
-            JSON.stringify({
-              ...props,
-              duration
-            })
-          );
+            const blob = new Blob(chunks, options);
+            const duration = Math.round((Date.now() - startTimestamp.current) / 1000);
+            formData.append('file', blob, filename);
+            formData.append(
+              'data',
+              JSON.stringify({
+                ...props,
+                duration
+              })
+            );
 
-          setIsTransCription(true);
-          try {
-            const result = await POST<string>('/v1/audio/transcriptions', formData, {
-              timeout: 60000,
-              headers: {
-                'Content-Type': 'multipart/form-data; charset=utf-8'
-              }
-            });
-            onFinish(result);
-          } catch (error) {
-            toast({
-              status: 'warning',
-              title: getErrText(error, t('common:common.speech.error tip'))
-            });
+            setIsTransCription(true);
+            try {
+              const result = await POST<string>('/v1/audio/transcriptions', formData, {
+                timeout: 60000,
+                headers: {
+                  'Content-Type': 'multipart/form-data; charset=utf-8'
+                }
+              });
+              onFinish(result);
+            } catch (error) {
+              toast({
+                status: 'warning',
+                title: getErrText(error, t('common:common.speech.error tip'))
+              });
+            }
+            setIsTransCription(false);
           }
+        };
+        mediaRecorder.current.onerror = (e) => {
+          if (timeIntervalRef.current) {
+            clearInterval(timeIntervalRef.current);
+          }
+          console.log('error', e);
+          setIsSpeaking(false);
+        };
+
+        // If onclick stop, stop speak
+        if (stopCalledRef.current) {
+          mediaRecorder.current.stop();
+        } else {
+          mediaRecorder.current.start();
         }
+      } catch (error) {
+        toast({
+          status: 'warning',
+          title: getErrText(error, 'Whisper error')
+        });
+        console.log(error);
+      }
+    },
+    [toast, t, props]
+  );
 
-        // close media stream
-        stream.getTracks().forEach((track) => track.stop());
-
-        setIsTransCription(false);
-        setIsSpeaking(false);
-      };
-
-      mediaRecorder.current.onerror = (e) => {
-        console.log('error', e);
-        setIsSpeaking(false);
-      };
-
-      mediaRecorder.current.start();
-    } catch (error) {
-      toast({
-        status: 'warning',
-        title: getErrText(error, 'Whisper error')
-      });
-      console.log(error);
-    }
-  };
-
-  const stopSpeak = (cancel = false) => {
+  const stopSpeak = useCallback((cancel = false) => {
     cancelWhisperSignal.current = cancel;
-    if (mediaRecorder.current) {
-      mediaRecorder.current?.stop();
-      clearInterval(intervalRef.current);
-    }
-  };
+    stopCalledRef.current = true;
 
+    if (timeIntervalRef.current) {
+      clearInterval(timeIntervalRef.current);
+    }
+
+    if (mediaRecorder.current && mediaRecorder.current.state !== 'inactive') {
+      mediaRecorder.current.stop();
+    }
+  }, []);
+
+  // Leave page, stop speak
   useEffect(() => {
     return () => {
-      clearInterval(intervalRef.current);
+      clearInterval(timeIntervalRef.current);
       if (mediaRecorder.current && mediaRecorder.current.state !== 'inactive') {
         mediaRecorder.current.stop();
       }
@@ -184,14 +273,15 @@ export const useSpeech = (props?: OutLinkChatAuthProps & { appId?: string }) =>
     if (audioSecond >= 60) {
       stopSpeak();
     }
-  }, [audioSecond]);
+  }, [audioSecond, stopSpeak]);
 
   return {
     startSpeak,
     stopSpeak,
     isSpeaking,
     isTransCription,
-    renderAudioGraph,
+    renderAudioGraphPc,
+    renderAudioGraphMobile,
     stream: mediaStream,
     speakingTimeString
   };

projects/app/src/web/core/dataset/api.ts

@@ -63,6 +63,17 @@ import type {
 import type { GetQuoteDataResponse } from '@/pages/api/core/dataset/data/getQuoteData';
 import type { GetQuotePermissionResponse } from '@/pages/api/core/dataset/data/getPermission';
 import type { GetQueueLenResponse } from '@/pages/api/core/dataset/training/getQueueLen';
+import type { updateTrainingDataBody } from '@/pages/api/core/dataset/training/updateTrainingData';
+import type {
+  getTrainingDataDetailBody,
+  getTrainingDataDetailResponse
+} from '@/pages/api/core/dataset/training/getTrainingDataDetail';
+import type { deleteTrainingDataBody } from '@/pages/api/core/dataset/training/deleteTrainingData';
+import type { getTrainingDetailResponse } from '@/pages/api/core/dataset/collection/trainingDetail';
+import type {
+  getTrainingErrorBody,
+  getTrainingErrorResponse
+} from '@/pages/api/core/dataset/training/getTrainingError';
 
 /* ======================== dataset ======================= */
 export const getDatasets = (data: GetDatasetListBody) =>
@@ -113,6 +124,10 @@ export const getDatasetCollectionPathById = (parentId: string) =>
   GET<ParentTreePathItemType[]>(`/core/dataset/collection/paths`, { parentId });
 export const getDatasetCollectionById = (id: string) =>
   GET<DatasetCollectionItemType>(`/core/dataset/collection/detail`, { id });
+export const getDatasetCollectionTrainingDetail = (collectionId: string) =>
+  GET<getTrainingDetailResponse>(`/core/dataset/collection/trainingDetail`, {
+    collectionId
+  });
 export const postDatasetCollection = (data: CreateDatasetCollectionParams) =>
   POST<string>(`/core/dataset/collection/create`, data);
 export const postCreateDatasetFileCollection = (data: FileIdCreateDatasetCollectionParams) =>
@@ -224,6 +239,15 @@ export const getPreviewChunks = (data: PostPreviewFilesChunksProps) =>
     timeout: 600000
   });
 
+export const deleteTrainingData = (data: deleteTrainingDataBody) =>
+  POST(`/core/dataset/training/deleteTrainingData`, data);
+export const updateTrainingData = (data: updateTrainingDataBody) =>
+  PUT(`/core/dataset/training/updateTrainingData`, data);
+export const getTrainingDataDetail = (data: getTrainingDataDetailBody) =>
+  POST<getTrainingDataDetailResponse>(`/core/dataset/training/getTrainingDataDetail`, data);
+export const getTrainingError = (data: getTrainingErrorBody) =>
+  POST<getTrainingErrorResponse>(`/core/dataset/training/getTrainingError`, data);
+
 /* ================== read source ======================== */
 export const getCollectionSource = (data: readCollectionSourceBody) =>
   POST<readCollectionSourceResponse>('/core/dataset/collection/read', data);

projects/app/src/web/core/dataset/constants.ts

@@ -2,13 +2,15 @@ import { defaultQAModels, defaultVectorModels } from '@fastgpt/global/core/ai/mo
 import {
   DatasetCollectionDataProcessModeEnum,
   DatasetCollectionTypeEnum,
-  DatasetTypeEnum
+  DatasetTypeEnum,
+  TrainingModeEnum
 } from '@fastgpt/global/core/dataset/constants';
 import type {
   DatasetCollectionItemType,
   DatasetItemType
 } from '@fastgpt/global/core/dataset/type.d';
 import { DatasetPermission } from '@fastgpt/global/support/permission/dataset/controller';
+import { i18nT } from '@fastgpt/web/i18n/utils';
 
 export const defaultDatasetDetail: DatasetItemType = {
   _id: '',
@@ -45,7 +47,6 @@ export const defaultCollectionDetail: DatasetCollectionItemType = {
     avatar: '/icon/logo.svg',
     name: '',
     intro: '',
-    status: 'active',
     vectorModel: defaultVectorModels[0].model,
     agentModel: defaultQAModels[0].model,
     inheritPermission: true
@@ -74,3 +75,34 @@ export const datasetTypeCourseMap: Record<`${DatasetTypeEnum}`, string> = {
   [DatasetTypeEnum.yuque]: '/docs/guide/knowledge_base/yuque_dataset/',
   [DatasetTypeEnum.externalFile]: ''
 };
+
+export const TrainingProcess = {
+  waiting: {
+    label: i18nT('dataset:process.Waiting'),
+    value: 'waiting'
+  },
+  parsing: {
+    label: i18nT('dataset:process.Parsing'),
+    value: 'parsing'
+  },
+  getQA: {
+    label: i18nT('dataset:process.Get QA'),
+    value: 'getQA'
+  },
+  imageIndex: {
+    label: i18nT('dataset:process.Image_Index'),
+    value: 'imageIndex'
+  },
+  autoIndex: {
+    label: i18nT('dataset:process.Auto_Index'),
+    value: 'autoIndex'
+  },
+  vectorizing: {
+    label: i18nT('dataset:process.Vectorizing'),
+    value: 'vectorizing'
+  },
+  isReady: {
+    label: i18nT('dataset:process.Is_Ready'),
+    value: 'isReady'
+  }
+};