V4.9.4 feature (#4470)

* Training status (#4424)

* dataset data training state (#4311)

* dataset data training state

* fix

* fix ts

* fix

* fix api format

* fix

* fix

* perf: count training

* format

* fix: dataset training state (#4417)

* fix

* add test

* fix

* fix

* fix test

* fix test

* perf: training count

* count

* loading status

---------

Co-authored-by: heheer <heheer@sealos.io>

* doc

* website sync feature (#4429)

* perf: introduce BullMQ for website sync (#4403)

* perf: introduce BullMQ for website sync

* feat: new redis module

* fix: remove graceful shutdown

* perf: improve UI in dataset detail

- Updated the "change" icon SVG file.
- Modified i18n strings.
- Added new i18n string "immediate_sync".
- Improved UI in dataset detail page, including button icons and
background colors.

* refactor: Add chunkSettings to DatasetSchema

* perf: website sync ux

* env template

* fix: clean up website dataset when updating chunk settings (#4420)

* perf: check setting updated

* perf: worker currency

* feat: init script for website sync refactor (#4425)

* website feature doc

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

* pro migration (#4388) (#4433)

* pro migration

* reuse customPdfParseType

Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>

* perf: remove loading ui

* feat: config chat file expired time

* Redis cache (#4436)

* perf: add Redis cache for vector counting (#4432)

* feat: cache

* perf: get cache key

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

* perf: mobile voice input (#4437)

* update:Mobile voice interaction (#4362)

* Add files via upload

* Add files via upload

* Update ollama.md

* Update ollama.md

* Add files via upload

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update ChatInput.tsx

* Update useSpeech.ts

* Update constants.ts

* Add files via upload

* Update ChatInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update useSpeech.ts

* Update ChatInput.tsx

* Add files via upload

* Update common.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update useSpeech.ts

* Update useSpeech.ts

* Update common.json

* Update common.json

* Update common.json

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update common.json

* Update chat.json

* Update VoiceInput.tsx

* Update ChatInput.tsx

* Update useSpeech.ts

* Update VoiceInput.tsx

* speech ui

* 优化语音输入组件，调整输入框显示逻辑，修复语音输入遮罩层样式，更新画布背景透明度，增强用户交互体验。 (#4435)

* perf: mobil voice input

---------

Co-authored-by: dreamer6680 <1468683855@qq.com>

* Test completion v2 (#4438)

* add v2 completions (#4364)

* add v2 completions

* completion config

* config version

* fix

* frontend

* doc

* fix

* fix: completions v2 api

---------

Co-authored-by: heheer <heheer@sealos.io>

* package

* Test mongo log (#4443)

* feat: mongodb-log (#4426)

* perf: mongo log

* feat: completions stop reasoner

* mongo db log

---------

Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>

* update doc

* Update doc

* fix external var ui (#4444)

* action

* fix: ts (#4458)

* preview doc action

add docs preview permission

update preview action

udpate action

* update doc (#4460)

* update preview action

* update doc

* remove

* update

* schema

* update mq export;perf: redis cache  (#4465)

* perf: redis cache

* update mq export

* perf: website sync error tip

* add error worker

* website sync ui (#4466)

* Updated the dynamic display of the voice input pop-up (#4469)

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* Update VoiceInput.tsx

* fix: voice input

---------

Co-authored-by: heheer <heheer@sealos.io>
Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>
Co-authored-by: gggaaallleee <91131304+gggaaallleee@users.noreply.github.com>
Co-authored-by: dreamer6680 <1468683855@qq.com>
Co-authored-by: Finley Ge <32237950+FinleyGe@users.noreply.github.com>

projects/app/src/pageComponents/dataset/detail/CollectionCard/WebsiteConfig.tsx

@@ -1,110 +1,215 @@
-import React from 'react';
 import MyModal from '@fastgpt/web/components/common/MyModal';
 import { useTranslation } from 'next-i18next';
-import { Box, Button, Input, Link, ModalBody, ModalFooter } from '@chakra-ui/react';
 import { strIsLink } from '@fastgpt/global/common/string/tools';
 import { useToast } from '@fastgpt/web/hooks/useToast';
 import { useForm } from 'react-hook-form';
 import { useConfirm } from '@fastgpt/web/hooks/useConfirm';
 import { getDocPath } from '@/web/common/system/doc';
 import { useSystemStore } from '@/web/common/system/useSystemStore';
+import { useMyStep } from '@fastgpt/web/hooks/useStep';
+import MyDivider from '@fastgpt/web/components/common/MyDivider';
+import React, { useRef } from 'react';
+import {
+  Box,
+  Link,
+  Input,
+  Button,
+  ModalBody,
+  ModalFooter,
+  Textarea,
+  Stack
+} from '@chakra-ui/react';
+import {
+  DataChunkSplitModeEnum,
+  DatasetCollectionDataProcessModeEnum
+} from '@fastgpt/global/core/dataset/constants';
+import { ChunkSettingModeEnum } from '@fastgpt/global/core/dataset/constants';
+import { Prompt_AgentQA } from '@fastgpt/global/core/ai/prompt/agent';
+import { useContextSelector } from 'use-context-selector';
+import { DatasetPageContext } from '@/web/core/dataset/context/datasetPageContext';
+import CollectionChunkForm, {
+  collectionChunkForm2StoreChunkData,
+  type CollectionChunkFormType
+} from '../Form/CollectionChunkForm';
+import { getLLMDefaultChunkSize } from '@fastgpt/global/core/dataset/training/utils';
+import { ChunkSettingsType } from '@fastgpt/global/core/dataset/type';
 
-type FormType = {
-  url?: string | undefined;
-  selector?: string | undefined;
+export type WebsiteConfigFormType = {
+  websiteConfig: {
+    url: string;
+    selector: string;
+  };
+  chunkSettings: ChunkSettingsType;
 };
 
 const WebsiteConfigModal = ({
   onClose,
-  onSuccess,
-  defaultValue = {
-    url: '',
-    selector: ''
-  }
+  onSuccess
 }: {
   onClose: () => void;
-  onSuccess: (data: FormType) => void;
-  defaultValue?: FormType;
+  onSuccess: (data: WebsiteConfigFormType) => void;
 }) => {
   const { t } = useTranslation();
   const { feConfigs } = useSystemStore();
   const { toast } = useToast();
-  const { register, handleSubmit } = useForm({
-    defaultValues: defaultValue
+  const steps = [
+    {
+      title: t('dataset:website_info')
+    },
+    {
+      title: t('dataset:params_config')
+    }
+  ];
+
+  const datasetDetail = useContextSelector(DatasetPageContext, (v) => v.datasetDetail);
+  const websiteConfig = datasetDetail.websiteConfig;
+  const chunkSettings = datasetDetail.chunkSettings;
+
+  const {
+    register: websiteInfoForm,
+    handleSubmit: websiteInfoHandleSubmit,
+    getValues: websiteInfoGetValues
+  } = useForm({
+    defaultValues: {
+      url: websiteConfig?.url || '',
+      selector: websiteConfig?.selector || ''
+    }
   });
-  const isEdit = !!defaultValue.url;
-  const confirmTip = isEdit
-    ? t('common:core.dataset.website.Confirm Update Tips')
-    : t('common:core.dataset.website.Confirm Create Tips');
+
+  const isEdit = !!websiteConfig?.url;
 
   const { ConfirmModal, openConfirm } = useConfirm({
     type: 'common'
   });
 
+  const { activeStep, goToPrevious, goToNext, MyStep } = useMyStep({
+    defaultStep: 0,
+    steps
+  });
+
+  const form = useForm<CollectionChunkFormType>({
+    defaultValues: {
+      trainingType: chunkSettings?.trainingType || DatasetCollectionDataProcessModeEnum.chunk,
+      imageIndex: chunkSettings?.imageIndex || false,
+      autoIndexes: chunkSettings?.autoIndexes || false,
+
+      chunkSettingMode: chunkSettings?.chunkSettingMode || ChunkSettingModeEnum.auto,
+      chunkSplitMode: chunkSettings?.chunkSplitMode || DataChunkSplitModeEnum.size,
+      embeddingChunkSize: chunkSettings?.chunkSize || 2000,
+      qaChunkSize: chunkSettings?.chunkSize || getLLMDefaultChunkSize(datasetDetail.agentModel),
+      indexSize: chunkSettings?.indexSize || datasetDetail.vectorModel?.defaultToken || 512,
+
+      chunkSplitter: chunkSettings?.chunkSplitter || '',
+      qaPrompt: chunkSettings?.qaPrompt || Prompt_AgentQA.description
+    }
+  });
+
   return (
     <MyModal
       isOpen
       iconSrc="core/dataset/websiteDataset"
       title={t('common:core.dataset.website.Config')}
       onClose={onClose}
-      maxW={'500px'}
+      w={'550px'}
     >
-      <ModalBody>
-        <Box fontSize={'sm'} color={'myGray.600'}>
-          {t('common:core.dataset.website.Config Description')}
-          {feConfigs?.docUrl && (
-            <Link
-              href={getDocPath('/docs/guide/knowledge_base/websync/')}
-              target="_blank"
-              textDecoration={'underline'}
-              fontWeight={'bold'}
+      <ModalBody w={'full'}>
+        <Stack w={'75%'} marginX={'auto'}>
+          <MyStep />
+        </Stack>
+        <MyDivider />
+        {activeStep == 0 && (
+          <>
+            <Box
+              fontSize={'xs'}
+              color={'myGray.900'}
+              bgColor={'blue.50'}
+              padding={'4'}
+              borderRadius={'8px'}
             >
-              {t('common:common.course.Read Course')}
-            </Link>
-          )}
-        </Box>
-        <Box mt={2}>
-          <Box>{t('common:core.dataset.website.Base Url')}</Box>
-          <Input
-            placeholder={t('common:core.dataset.collection.Website Link')}
-            {...register('url', {
-              required: true
-            })}
-          />
-        </Box>
-        <Box mt={3}>
-          <Box>
-            {t('common:core.dataset.website.Selector')}({t('common:common.choosable')})
-          </Box>
-          <Input {...register('selector')} placeholder="body .content #document" />
-        </Box>
+              {t('common:core.dataset.website.Config Description')}
+              {feConfigs?.docUrl && (
+                <Link
+                  href={getDocPath('/docs/guide/knowledge_base/websync/')}
+                  target="_blank"
+                  textDecoration={'underline'}
+                  color={'blue.700'}
+                >
+                  {t('common:common.course.Read Course')}
+                </Link>
+              )}
+            </Box>
+            <Box mt={2}>
+              <Box>{t('common:core.dataset.website.Base Url')}</Box>
+              <Input
+                placeholder={t('common:core.dataset.collection.Website Link')}
+                {...websiteInfoForm('url', {
+                  required: true
+                })}
+              />
+            </Box>
+            <Box mt={3}>
+              <Box>
+                {t('common:core.dataset.website.Selector')}({t('common:common.choosable')})
+              </Box>
+              <Input {...websiteInfoForm('selector')} placeholder="body .content #document" />
+            </Box>
+          </>
+        )}
+        {activeStep == 1 && <CollectionChunkForm form={form} />}
       </ModalBody>
       <ModalFooter>
-        <Button variant={'whiteBase'} onClick={onClose}>
-          {t('common:common.Close')}
-        </Button>
-        <Button
-          ml={2}
-          onClick={handleSubmit((data) => {
-            if (!data.url) return;
-            // check is link
-            if (!strIsLink(data.url)) {
-              return toast({
-                status: 'warning',
-                title: t('common:common.link.UnValid')
-              });
-            }
-            openConfirm(
-              () => {
-                onSuccess(data);
-              },
-              undefined,
-              confirmTip
-            )();
-          })}
-        >
-          {t('common:core.dataset.website.Start Sync')}
-        </Button>
+        {activeStep == 0 && (
+          <>
+            <Button variant={'whiteBase'} onClick={onClose}>
+              {t('common:common.Close')}
+            </Button>
+            <Button
+              ml={2}
+              onClick={websiteInfoHandleSubmit((data) => {
+                if (!data.url) return;
+                // check is link
+                if (!strIsLink(data.url)) {
+                  return toast({
+                    status: 'warning',
+                    title: t('common:common.link.UnValid')
+                  });
+                }
+                goToNext();
+              })}
+            >
+              {t('common:common.Next Step')}
+            </Button>
+          </>
+        )}
+        {activeStep == 1 && (
+          <>
+            <Button variant={'whiteBase'} onClick={goToPrevious}>
+              {t('common:common.Last Step')}
+            </Button>
+            <Button
+              ml={2}
+              onClick={form.handleSubmit((data) => {
+                openConfirm(
+                  () =>
+                    onSuccess({
+                      websiteConfig: websiteInfoGetValues(),
+                      chunkSettings: collectionChunkForm2StoreChunkData({
+                        ...data,
+                        agentModel: datasetDetail.agentModel,
+                        vectorModel: datasetDetail.vectorModel
+                      })
+                    }),
+                  undefined,
+                  isEdit
+                    ? t('common:core.dataset.website.Confirm Update Tips')
+                    : t('common:core.dataset.website.Confirm Create Tips')
+                )();
+              })}
+            >
+              {t('common:core.dataset.website.Start Sync')}
+            </Button>
+          </>
+        )}
       </ModalFooter>
       <ConfirmModal />
     </MyModal>
@@ -112,3 +217,42 @@ const WebsiteConfigModal = ({
 };
 
 export default WebsiteConfigModal;
+
+const PromptTextarea = ({
+  defaultValue,
+  onChange,
+  onClose
+}: {
+  defaultValue: string;
+  onChange: (e: string) => void;
+  onClose: () => void;
+}) => {
+  const ref = useRef<HTMLTextAreaElement>(null);
+  const { t } = useTranslation();
+
+  return (
+    <MyModal
+      isOpen
+      title={t('common:core.dataset.import.Custom prompt')}
+      iconSrc="modal/edit"
+      w={'600px'}
+      onClose={onClose}
+    >
+      <ModalBody whiteSpace={'pre-wrap'} fontSize={'sm'} px={[3, 6]} pt={[3, 6]}>
+        <Textarea ref={ref} rows={8} fontSize={'sm'} defaultValue={defaultValue} />
+        <Box>{Prompt_AgentQA.fixedText}</Box>
+      </ModalBody>
+      <ModalFooter>
+        <Button
+          onClick={() => {
+            const val = ref.current?.value || Prompt_AgentQA.description;
+            onChange(val);
+            onClose();
+          }}
+        >
+          {t('common:common.Confirm')}
+        </Button>
+      </ModalFooter>
+    </MyModal>
+  );
+};