v4.5.1 (#417)

projects/app/src/components/ChatBox/FeedbackModal.tsx

@@ -3,7 +3,7 @@ import { ModalBody, Textarea, ModalFooter, Button } from '@chakra-ui/react';
 import MyModal from '../MyModal';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { useTranslation } from 'next-i18next';
-import { userUpdateChatFeedback } from '@/web/core/api/chat';
+import { userUpdateChatFeedback } from '@/web/core/chat/api';
 
 const FeedbackModal = ({
   chatItemId,

projects/app/src/components/ChatBox/MarkModal.tsx

@@ -0,0 +1,16 @@
+import React, { useState } from 'react';
+
+const MarkModal = () => {
+  const [adminMarkData, setAdminMarkData] = useState<{
+    chatItemId: string;
+    dataId?: string;
+    datasetId?: string;
+    collectionId?: string;
+    q: string;
+    a: string;
+  }>();
+
+  return <div>MarkModal</div>;
+};
+
+export default MarkModal;

projects/app/src/components/ChatBox/QuoteModal.tsx

@@ -1,23 +1,24 @@
 import React, { useCallback, useMemo, useState } from 'react';
 import { ModalBody, Box, useTheme, Flex, Progress } from '@chakra-ui/react';
-import { getDatasetDataItemById } from '@/web/core/api/dataset';
+import { getDatasetDataItemById } from '@/web/core/dataset/api';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import { useToast } from '@/web/common/hooks/useToast';
-import { getErrText } from '@/utils/tools';
-import { QuoteItemType } from '@/types/chat';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 import MyIcon from '@/components/Icon';
-import InputDataModal, { RawFileText } from '@/pages/kb/detail/components/InputDataModal';
+import InputDataModal, {
+  RawSourceText,
+  type InputDataType
+} from '@/pages/dataset/detail/components/InputDataModal';
 import MyModal from '../MyModal';
 import { useTranslation } from 'react-i18next';
 import { useRouter } from 'next/router';
+import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 
 const QuoteModal = ({
-  onUpdateQuote,
   rawSearch = [],
   onClose
 }: {
-  onUpdateQuote: (quoteId: string, sourceText?: string) => Promise<void>;
-  rawSearch: QuoteItemType[];
+  rawSearch: SearchDataResponseItemType[];
   onClose: () => void;
 }) => {
   const { t } = useTranslation();
@@ -25,7 +26,7 @@ const QuoteModal = ({
   const router = useRouter();
   const { toast } = useToast();
   const { setIsLoading, Loading } = useLoading();
-  const [editDataItem, setEditDataItem] = useState<QuoteItemType>();
+  const [editInputData, setEditInputData] = useState<InputDataType>();
 
   const isShare = useMemo(() => router.pathname === '/chat/share', [router.pathname]);
 
@@ -33,18 +34,17 @@ const QuoteModal = ({
    * click edit, get new kbDataItem
    */
   const onclickEdit = useCallback(
-    async (item: QuoteItemType) => {
+    async (item: InputDataType) => {
       if (!item.id) return;
       try {
         setIsLoading(true);
         const data = await getDatasetDataItemById(item.id);
 
         if (!data) {
-          onUpdateQuote(item.id, '已删除');
           throw new Error('该数据已被删除');
         }
 
-        setEditDataItem(data);
+        setEditInputData(data);
       } catch (err) {
         toast({
           status: 'warning',
@@ -53,7 +53,7 @@ const QuoteModal = ({
       }
       setIsLoading(false);
     },
-    [setIsLoading, toast, onUpdateQuote]
+    [setIsLoading, toast]
   );
 
   return (
@@ -94,10 +94,7 @@ const QuoteModal = ({
             >
               {!isShare && (
                 <Flex alignItems={'center'} mb={1}>
-                  <RawFileText
-                    filename={item.source || t('common.Unknow') || 'Unknow'}
-                    fileId={item.file_id}
-                  />
+                  <RawSourceText sourceName={item.sourceName} sourceId={item.sourceId} />
                   <Box flex={'1'} />
                   {item.score && (
                     <>
@@ -150,16 +147,17 @@ const QuoteModal = ({
         </ModalBody>
         <Loading fixed={false} />
       </MyModal>
-      {editDataItem && (
+      {editInputData && editInputData.id && (
         <InputDataModal
-          onClose={() => setEditDataItem(undefined)}
-          onSuccess={() => onUpdateQuote(editDataItem.id)}
-          onDelete={() => onUpdateQuote(editDataItem.id, '已删除')}
-          kbId={editDataItem.kb_id}
-          defaultValues={{
-            ...editDataItem,
-            dataId: editDataItem.id
+          onClose={() => setEditInputData(undefined)}
+          onSuccess={() => {
+            console.log('更新引用成功');
           }}
+          onDelete={() => {
+            console.log('删除引用成功');
+          }}
+          datasetId={editInputData.datasetId}
+          defaultValues={editInputData}
         />
       )}
     </>

projects/app/src/components/ChatBox/ReadFeedbackModal.tsx

@@ -3,7 +3,7 @@ import { ModalBody, ModalFooter, Button } from '@chakra-ui/react';
 import MyModal from '../MyModal';
 import { useRequest } from '@/web/common/hooks/useRequest';
 import { useTranslation } from 'next-i18next';
-import { userUpdateChatFeedback } from '@/web/core/api/chat';
+import { userUpdateChatFeedback } from '@/web/core/chat/api';
 
 const ReadFeedbackModal = ({
   chatItemId,
@@ -39,14 +39,14 @@ const ReadFeedbackModal = ({
     <MyModal isOpen={true} onClose={onClose} title={t('chat.Feedback Modal')}>
       <ModalBody>{content}</ModalBody>
       <ModalFooter>
+        <Button mr={2} isLoading={isLoading} variant={'base'} onClick={mutate}>
+          {t('chat.Feedback Close')}
+        </Button>
         {!isMarked && (
-          <Button variant={'base'} mr={2} onClick={onMark}>
+          <Button mr={2} onClick={onMark}>
             {t('chat.Feedback Mark')}
           </Button>
         )}
-        <Button isLoading={isLoading} onClick={mutate}>
-          {t('chat.Feedback Close')}
-        </Button>
       </ModalFooter>
     </MyModal>
   );

projects/app/src/components/ChatBox/ResponseTags.tsx

@@ -1,8 +1,9 @@
-import React, { useCallback, useMemo, useState } from 'react';
-import { ChatHistoryItemResType, ChatItemType, QuoteItemType } from '@/types/chat';
+import React, { useMemo, useState } from 'react';
+import { ChatHistoryItemResType, ChatItemType } from '@/types/chat';
 import { Flex, BoxProps, useDisclosure } from '@chakra-ui/react';
 import { useTranslation } from 'react-i18next';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type';
 import dynamic from 'next/dynamic';
 import Tag from '../Tag';
 import MyTooltip from '../MyTooltip';
@@ -13,9 +14,9 @@ const ContextModal = dynamic(() => import('./ContextModal'), { ssr: false });
 const WholeResponseModal = dynamic(() => import('./WholeResponseModal'), { ssr: false });
 
 const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemResType[] }) => {
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   const { t } = useTranslation();
-  const [quoteModalData, setQuoteModalData] = useState<QuoteItemType[]>();
+  const [quoteModalData, setQuoteModalData] = useState<SearchDataResponseItemType[]>();
   const [contextModalData, setContextModalData] = useState<ChatItemType[]>();
   const {
     isOpen: isOpenWholeModal,
@@ -37,14 +38,12 @@ const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemRes
         .filter((item) => item.moduleType === FlowModuleTypeEnum.chatNode)
         .map((item) => item.quoteList)
         .flat()
-        .filter((item) => item) as QuoteItemType[],
+        .filter((item) => item) as SearchDataResponseItemType[],
       historyPreview: chatData?.historyPreview,
       runningTime: +responseData.reduce((sum, item) => sum + (item.runningTime || 0), 0).toFixed(2)
     };
   }, [responseData]);
 
-  const updateQuote = useCallback(async (quoteId: string, sourceText?: string) => {}, []);
-
   const TagStyles: BoxProps = {
     mr: 2,
     bg: 'transparent'
@@ -100,11 +99,7 @@ const ResponseTags = ({ responseData = [] }: { responseData?: ChatHistoryItemRes
       </MyTooltip>
 
       {!!quoteModalData && (
-        <QuoteModal
-          rawSearch={quoteModalData}
-          onUpdateQuote={updateQuote}
-          onClose={() => setQuoteModalData(undefined)}
-        />
+        <QuoteModal rawSearch={quoteModalData} onClose={() => setQuoteModalData(undefined)} />
       )}
       {!!contextModalData && (
         <ContextModal context={contextModalData} onClose={() => setContextModalData(undefined)} />

projects/app/src/components/ChatBox/SelectDataset.tsx

@@ -1,114 +0,0 @@
-import React, { useState } from 'react';
-import { ModalBody, useTheme, ModalFooter, Button, Box, Card, Flex, Grid } from '@chakra-ui/react';
-import { useTranslation } from 'next-i18next';
-import { useToast } from '@/web/common/hooks/useToast';
-import Avatar from '../Avatar';
-import MyIcon from '@/components/Icon';
-import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
-import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
-
-const SelectDataset = ({
-  isOpen,
-  onSuccess,
-  onClose
-}: {
-  isOpen: boolean;
-  onSuccess: (kbId: string) => void;
-  onClose: () => void;
-}) => {
-  const { t } = useTranslation();
-  const theme = useTheme();
-  const { toast } = useToast();
-  const [selectedId, setSelectedId] = useState<string>();
-  const { paths, parentId, setParentId, datasets } = useDatasetSelect();
-
-  return (
-    <DatasetSelectModal
-      isOpen={isOpen}
-      paths={paths}
-      onClose={onClose}
-      parentId={parentId}
-      setParentId={setParentId}
-      tips={t('chat.Select Mark Kb Desc')}
-    >
-      <ModalBody flex={'1 0 0'} overflowY={'auto'}>
-        <Grid
-          gridTemplateColumns={['repeat(1,1fr)', 'repeat(2,1fr)', 'repeat(3,1fr)']}
-          gridGap={3}
-          userSelect={'none'}
-        >
-          {datasets.map((item) =>
-            (() => {
-              const selected = selectedId === item._id;
-              return (
-                <Card
-                  key={item._id}
-                  p={3}
-                  border={theme.borders.base}
-                  boxShadow={'sm'}
-                  h={'80px'}
-                  cursor={'pointer'}
-                  _hover={{
-                    boxShadow: 'md'
-                  }}
-                  {...(selected
-                    ? {
-                        bg: 'myBlue.300'
-                      }
-                    : {})}
-                  onClick={() => {
-                    if (item.type === DatasetTypeEnum.folder) {
-                      setParentId(item._id);
-                    } else {
-                      setSelectedId(item._id);
-                    }
-                  }}
-                >
-                  <Flex alignItems={'center'} h={'38px'}>
-                    <Avatar src={item.avatar} w={['24px', '28px', '32px']}></Avatar>
-                    <Box ml={3} fontWeight={'bold'} fontSize={['md', 'lg', 'xl']}>
-                      {item.name}
-                    </Box>
-                  </Flex>
-                  <Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
-                    <MyIcon mr={1} name="kbTest" w={'12px'} />
-                    <Box color={'myGray.500'}>{item.vectorModel.name}</Box>
-                  </Flex>
-                </Card>
-              );
-            })()
-          )}
-        </Grid>
-        {datasets.length === 0 && (
-          <Flex mt={5} flexDirection={'column'} alignItems={'center'}>
-            <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
-            <Box mt={2} color={'myGray.500'}>
-              这个目录已经没东西可选了~
-            </Box>
-          </Flex>
-        )}
-      </ModalBody>
-      <ModalFooter>
-        <Button variant={'base'} mr={2} onClick={onClose}>
-          {t('Cancel')}
-        </Button>
-        <Button
-          onClick={() => {
-            if (!selectedId) {
-              return toast({
-                status: 'warning',
-                title: t('Select value is empty')
-              });
-            }
-
-            onSuccess(selectedId);
-          }}
-        >
-          {t('Confirm')}
-        </Button>
-      </ModalFooter>
-    </DatasetSelectModal>
-  );
-};
-
-export default SelectDataset;

projects/app/src/components/ChatBox/SelectMarkCollection.tsx

@@ -0,0 +1,195 @@
+import React, { useState } from 'react';
+import { ModalBody, useTheme, ModalFooter, Button, Box, Card, Flex, Grid } from '@chakra-ui/react';
+import { useTranslation } from 'next-i18next';
+import Avatar from '../Avatar';
+import MyIcon from '@/components/Icon';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import DatasetSelectModal, { useDatasetSelect } from '@/components/core/dataset/SelectModal';
+import dynamic from 'next/dynamic';
+import { MarkDataType } from '@/global/core/dataset/type';
+import SelectCollections from '@/web/core/dataset/components/SelectCollections';
+
+const InputDataModal = dynamic(() => import('@/pages/dataset/detail/components/InputDataModal'));
+
+export type AdminMarkType = {
+  dataId?: string;
+  datasetId?: string;
+  collectionId?: string;
+  q: string;
+  a?: string;
+};
+
+const SelectMarkCollection = ({
+  adminMarkData,
+  setAdminMarkData,
+  onSuccess,
+  onClose
+}: {
+  adminMarkData: AdminMarkType;
+  setAdminMarkData: (e: AdminMarkType) => void;
+  onClose: () => void;
+  onSuccess: (adminFeedback: MarkDataType) => void;
+}) => {
+  const { t } = useTranslation();
+  const theme = useTheme();
+  const [selectedDatasetId, setSelectedDatasetId] = useState<string>();
+  const [selectedDatasetCollectionIds, setSelectedDatasetCollectionIds] = useState<string[]>([]);
+  const { paths, parentId, setParentId, datasets, isLoading } = useDatasetSelect();
+
+  return (
+    <>
+      {/* select dataset */}
+      {!adminMarkData.datasetId && (
+        <DatasetSelectModal
+          isOpen
+          paths={paths}
+          onClose={onClose}
+          parentId={parentId}
+          setParentId={setParentId}
+          tips={t('chat.Select Mark Kb Desc')}
+        >
+          <ModalBody flex={'1 0 0'} overflowY={'auto'}>
+            <Grid
+              gridTemplateColumns={['repeat(1,1fr)', 'repeat(2,1fr)', 'repeat(3,1fr)']}
+              gridGap={3}
+              userSelect={'none'}
+            >
+              {datasets.map((item) =>
+                (() => {
+                  const selected = selectedDatasetId === item._id;
+                  return (
+                    <Card
+                      key={item._id}
+                      p={3}
+                      border={theme.borders.base}
+                      boxShadow={'sm'}
+                      h={'80px'}
+                      cursor={'pointer'}
+                      _hover={{
+                        boxShadow: 'md'
+                      }}
+                      {...(selected
+                        ? {
+                            bg: 'myBlue.300'
+                          }
+                        : {})}
+                      onClick={() => {
+                        if (item.type === DatasetTypeEnum.folder) {
+                          setParentId(item._id);
+                        } else {
+                          setSelectedDatasetId(item._id);
+                        }
+                      }}
+                    >
+                      <Flex alignItems={'center'} h={'38px'}>
+                        <Avatar src={item.avatar} w={['24px', '28px', '32px']}></Avatar>
+                        <Box ml={3} fontWeight={'bold'} fontSize={['md', 'lg', 'xl']}>
+                          {item.name}
+                        </Box>
+                      </Flex>
+                      <Flex justifyContent={'flex-end'} alignItems={'center'} fontSize={'sm'}>
+                        <MyIcon mr={1} name="kbTest" w={'12px'} />
+                        <Box color={'myGray.500'}>{item.vectorModel.name}</Box>
+                      </Flex>
+                    </Card>
+                  );
+                })()
+              )}
+            </Grid>
+            {datasets.length === 0 && (
+              <Flex mt={'10vh'} flexDirection={'column'} alignItems={'center'}>
+                <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
+                <Box mt={2} color={'myGray.500'}>
+                  这个目录已经没东西可选了~
+                </Box>
+              </Flex>
+            )}
+          </ModalBody>
+          <ModalFooter>
+            <Button
+              isLoading={isLoading}
+              isDisabled={!selectedDatasetId}
+              onClick={() => {
+                setAdminMarkData({ ...adminMarkData, datasetId: selectedDatasetId });
+              }}
+            >
+              {t('common.Next Step')}
+            </Button>
+          </ModalFooter>
+        </DatasetSelectModal>
+      )}
+
+      {/* select collection */}
+      {adminMarkData.datasetId && !adminMarkData.collectionId && (
+        <SelectCollections
+          datasetId={adminMarkData.datasetId}
+          type={'collection'}
+          title={t('dataset.collections.Select One Collection To Store')}
+          onClose={onClose}
+          onChange={({ collectionIds }) => {
+            setSelectedDatasetCollectionIds(collectionIds);
+          }}
+          CustomFooter={
+            <ModalFooter>
+              <Button
+                variant={'base'}
+                mr={2}
+                onClick={() => {
+                  setAdminMarkData({
+                    ...adminMarkData,
+                    datasetId: undefined
+                  });
+                }}
+              >
+                {t('common.Last Step')}
+              </Button>
+              <Button
+                isDisabled={selectedDatasetCollectionIds.length === 0}
+                onClick={() => {
+                  setAdminMarkData({
+                    ...adminMarkData,
+                    collectionId: selectedDatasetCollectionIds[0]
+                  });
+                }}
+              >
+                {t('common.Next Step')}
+              </Button>
+            </ModalFooter>
+          }
+        />
+      )}
+
+      {/* input data */}
+      {adminMarkData.datasetId && adminMarkData.collectionId && (
+        <InputDataModal
+          onClose={onClose}
+          datasetId={adminMarkData.datasetId}
+          defaultValues={{
+            id: adminMarkData.dataId,
+            datasetId: adminMarkData.datasetId,
+            collectionId: adminMarkData.collectionId,
+            sourceName: '手动标注',
+            q: adminMarkData.q,
+            a: adminMarkData.a
+          }}
+          onSuccess={(data) => {
+            if (!data.q || !adminMarkData.datasetId || !adminMarkData.collectionId || !data.id) {
+              return onClose();
+            }
+
+            onSuccess({
+              dataId: data.id,
+              datasetId: adminMarkData.datasetId,
+              collectionId: adminMarkData.collectionId,
+              q: data.q,
+              a: data.a
+            });
+            onClose();
+          }}
+        />
+      )}
+    </>
+  );
+};
+
+export default React.memo(SelectMarkCollection);

projects/app/src/components/ChatBox/WholeResponseModal.tsx

@@ -8,7 +8,7 @@ import Tabs from '../Tabs';
 import MyModal from '../MyModal';
 import MyTooltip from '../MyTooltip';
 import { QuestionOutlineIcon } from '@chakra-ui/icons';
-import { formatPrice } from '@fastgpt/common/bill/index';
+import { formatPrice } from '@fastgpt/global/common/bill/tools';
 
 function Row({ label, value }: { label: string; value?: string | number | React.ReactNode }) {
   const theme = useTheme();

projects/app/src/components/ChatBox/index.module.scss

@@ -10,25 +10,6 @@
   }
 }
 
-.newChat {
-  .modelListContainer {
-    height: 0;
-    overflow: hidden;
-  }
-  .modelList {
-    border-radius: 6px;
-  }
-  &:hover {
-    .modelListContainer {
-      height: 60vh;
-    }
-    .modelList {
-      box-shadow: 0 0 5px rgba($color: #000000, $alpha: 0.05);
-      border: 1px solid #dee0e2;
-    }
-  }
-}
-
 .statusAnimation {
   animation: statusBox 0.8s linear infinite alternate;
 }

projects/app/src/components/ChatBox/index.tsx

@@ -8,6 +8,7 @@ import React, {
   ForwardedRef,
   useEffect
 } from 'react';
+import Script from 'next/script';
 import { throttle } from 'lodash';
 import {
   ChatHistoryItemResType,
@@ -17,7 +18,7 @@ import {
 } from '@/types/chat';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useAudioPlay } from '@/web/common/utils/voice';
-import { getErrText } from '@/utils/tools';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useCopyData } from '@/web/common/hooks/useCopyData';
 import {
   Box,
@@ -30,7 +31,7 @@ import {
   BoxProps,
   FlexProps
 } from '@chakra-ui/react';
-import { feConfigs } from '@/web/common/store/static';
+import { feConfigs } from '@/web/common/system/staticData';
 import { eventBus } from '@/web/common/utils/eventbus';
 import { adaptChat2GptMessages } from '@/utils/common/adapt/message';
 import { useMarkdown } from '@/web/common/hooks/useMarkdown';
@@ -38,14 +39,15 @@ import { AppModuleItemType } from '@/types/app';
 import { VariableInputEnum } from '@/constants/app';
 import { useForm } from 'react-hook-form';
 import type { MessageItemType } from '@/types/core/chat/type';
-import { fileDownload } from '@/web/common/utils/file';
+import { fileDownload } from '@/web/common/file/utils';
 import { htmlTemplate } from '@/constants/common';
 import { useRouter } from 'next/router';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { TaskResponseKeyEnum } from '@/constants/chat';
 import { useTranslation } from 'react-i18next';
 import { customAlphabet } from 'nanoid';
-import { userUpdateChatFeedback, adminUpdateChatFeedback } from '@/web/core/api/chat';
+import { adminUpdateChatFeedback, userUpdateChatFeedback } from '@/web/core/chat/api';
+import type { AdminMarkType } from './SelectMarkCollection';
 
 import MyIcon from '@/components/Icon';
 import Avatar from '@/components/Avatar';
@@ -56,14 +58,11 @@ import dynamic from 'next/dynamic';
 const ResponseTags = dynamic(() => import('./ResponseTags'));
 const FeedbackModal = dynamic(() => import('./FeedbackModal'));
 const ReadFeedbackModal = dynamic(() => import('./ReadFeedbackModal'));
-const SelectDataset = dynamic(() => import('./SelectDataset'));
-const InputDataModal = dynamic(() => import('@/pages/kb/detail/components/InputDataModal'));
+const SelectMarkCollection = dynamic(() => import('./SelectMarkCollection'));
 
 import styles from './index.module.scss';
-import Script from 'next/script';
-import { postQuestionGuide } from '@/web/core/api/ai';
-import { splitGuideModule } from './utils';
-import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
+import { postQuestionGuide } from '@/web/core/ai/api';
+import { splitGuideModule } from '@/global/core/app/modules/utils';
 
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24);
 
@@ -131,7 +130,7 @@ const ChatBox = (
   const router = useRouter();
   const { t } = useTranslation();
   const { toast } = useToast();
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   const TextareaDom = useRef<HTMLTextAreaElement>(null);
   const chatController = useRef(new AbortController());
   const questionGuideController = useRef(new AbortController());
@@ -147,14 +146,7 @@ const ChatBox = (
     content: string;
     isMarked: boolean;
   }>();
-  const [adminMarkData, setAdminMarkData] = useState<{
-    // mark modal data
-    kbId?: string;
-    chatItemId: string;
-    dataId?: string;
-    q: string;
-    a: string;
-  }>();
+  const [adminMarkData, setAdminMarkData] = useState<AdminMarkType & { chatItemId: string }>();
   const [questionGuides, setQuestionGuide] = useState<string[]>([]);
 
   const isChatting = useMemo(
@@ -674,10 +666,11 @@ const ChatBox = (
                                 if (item.adminFeedback) {
                                   setAdminMarkData({
                                     chatItemId: item.dataId,
-                                    kbId: item.adminFeedback.kbId,
+                                    datasetId: item.adminFeedback.datasetId,
+                                    collectionId: item.adminFeedback.collectionId,
                                     dataId: item.adminFeedback.dataId,
-                                    q: chatHistory[index - 1]?.value || '',
-                                    a: item.adminFeedback.content
+                                    q: item.adminFeedback.q || chatHistory[index - 1]?.value || '',
+                                    a: item.adminFeedback.a
                                   });
                                 } else {
                                   setAdminMarkData({
@@ -798,7 +791,9 @@ const ChatBox = (
                               </Box>
                               <Box h={'1px'} bg={'myGray.300'} flex={'1'} />
                             </Flex>
-                            <Box>{item.adminFeedback.content}</Box>
+                            <Box whiteSpace={'pre'}>{`${item.adminFeedback.q || ''}${
+                              item.adminFeedback.a ? `\n${item.adminFeedback.a}` : ''
+                            }`}</Box>
                           </Box>
                         )}
                       </Card>
@@ -940,75 +935,44 @@ const ChatBox = (
         />
       )}
       {/* admin mark data */}
-      {showMarkIcon && (
-        <>
-          {/* select one dataset to insert markData */}
-          <SelectDataset
-            isOpen={!!adminMarkData && !adminMarkData.kbId}
-            onClose={() => setAdminMarkData(undefined)}
-            // @ts-ignore
-            onSuccess={(kbId) => setAdminMarkData((state) => ({ ...state, kbId }))}
-          />
+      {!!adminMarkData && (
+        <SelectMarkCollection
+          adminMarkData={adminMarkData}
+          setAdminMarkData={(e) => setAdminMarkData({ ...e, chatItemId: adminMarkData.chatItemId })}
+          onClose={() => setAdminMarkData(undefined)}
+          onSuccess={(adminFeedback) => {
+            adminUpdateChatFeedback({
+              chatItemId: adminMarkData.chatItemId,
+              ...adminFeedback
+            });
+            // update dom
+            setChatHistory((state) =>
+              state.map((chatItem) =>
+                chatItem.dataId === adminMarkData.chatItemId
+                  ? {
+                      ...chatItem,
+                      adminFeedback
+                    }
+                  : chatItem
+              )
+            );
 
-          {/* edit markData modal */}
-          {adminMarkData && adminMarkData.kbId && (
-            <InputDataModal
-              onClose={() => setAdminMarkData(undefined)}
-              onSuccess={async (data) => {
-                if (!adminMarkData.kbId || !data.dataId) {
-                  return setAdminMarkData(undefined);
-                }
-                const adminFeedback = {
-                  kbId: adminMarkData.kbId,
-                  dataId: data.dataId,
-                  content: data.a
-                };
-
-                // update dom
-                setChatHistory((state) =>
-                  state.map((chatItem) =>
-                    chatItem.dataId === adminMarkData.chatItemId
-                      ? {
-                          ...chatItem,
-                          adminFeedback
-                        }
-                      : chatItem
-                  )
-                );
-                // request to update adminFeedback
-                try {
-                  adminUpdateChatFeedback({
-                    chatItemId: adminMarkData.chatItemId,
-                    ...adminFeedback
-                  });
-
-                  if (readFeedbackData) {
-                    userUpdateChatFeedback({
-                      chatItemId: readFeedbackData.chatItemId,
-                      userFeedback: undefined
-                    });
-                    setChatHistory((state) =>
-                      state.map((chatItem) =>
-                        chatItem.dataId === readFeedbackData.chatItemId
-                          ? { ...chatItem, userFeedback: undefined }
-                          : chatItem
-                      )
-                    );
-                    setReadFeedbackData(undefined);
-                  }
-                } catch (error) {}
-                setAdminMarkData(undefined);
-              }}
-              kbId={adminMarkData.kbId}
-              defaultValues={{
-                dataId: adminMarkData.dataId,
-                q: adminMarkData.q,
-                a: adminMarkData.a,
-                file_id: DatasetSpecialIdEnum.mark
-              }}
-            />
-          )}
-        </>
+            if (readFeedbackData) {
+              userUpdateChatFeedback({
+                chatItemId: readFeedbackData.chatItemId,
+                userFeedback: undefined
+              });
+              setChatHistory((state) =>
+                state.map((chatItem) =>
+                  chatItem.dataId === readFeedbackData.chatItemId
+                    ? { ...chatItem, userFeedback: undefined }
+                    : chatItem
+                )
+              );
+              setReadFeedbackData(undefined);
+            }
+          }}
+        />
       )}
     </Flex>
   );

projects/app/src/components/DateRangePicker/index.module.scss

@@ -1,4 +0,0 @@
-.datePicker {
-  --rdp-background-color: #d6e8ff;
-  --rdp-accent-color: #0000ff;
-}

projects/app/src/components/DateRangePicker/index.tsx

@@ -4,7 +4,6 @@ import { addDays, format } from 'date-fns';
 import { type DateRange, DayPicker } from 'react-day-picker';
 import MyIcon from '../Icon';
 import 'react-day-picker/dist/style.css';
-import styles from './index.module.scss';
 import zhCN from 'date-fns/locale/zh-CN';
 
 const DateRangePicker = ({
@@ -59,6 +58,10 @@ const DateRangePicker = ({
         <Card
           position={'absolute'}
           zIndex={1}
+          css={{
+            '--rdp-background-color': '#d6e8ff',
+            ' --rdp-accent-color': '#0000ff'
+          }}
           {...(position === 'top'
             ? {
                 bottom: '40px'
@@ -69,7 +72,6 @@ const DateRangePicker = ({
             locale={zhCN}
             id="test"
             mode="range"
-            className={styles.datePicker}
             defaultMonth={defaultDate.to}
             selected={range}
             disabled={[

projects/app/src/components/EmptyTip/index.tsx

@@ -0,0 +1,20 @@
+import React from 'react';
+import { Flex, Box, FlexProps } from '@chakra-ui/react';
+import MyIcon from '../Icon';
+
+type Props = FlexProps & {
+  text?: string | null;
+};
+
+const EmptyTip = ({ text, ...props }: Props) => {
+  return (
+    <Flex mt={5} flexDirection={'column'} alignItems={'center'} pt={'10vh'} {...props}>
+      <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
+      <Box mt={2} color={'myGray.500'}>
+        {text || '没有什么数据噢~'}
+      </Box>
+    </Flex>
+  );
+};
+
+export default EmptyTip;

projects/app/src/components/Layout/auth.tsx

@@ -1,7 +1,7 @@
 import React from 'react';
 import { useRouter } from 'next/router';
 import { useToast } from '@chakra-ui/react';
-import { useUserStore } from '@/web/support/store/user';
+import { useUserStore } from '@/web/support/user/useUserStore';
 import { useQuery } from '@tanstack/react-query';
 
 const unAuthPage: { [key: string]: boolean } = {

projects/app/src/components/Layout/index.tsx

@@ -2,14 +2,14 @@ import React, { useEffect, useMemo } from 'react';
 import { Box, useColorMode, Flex } from '@chakra-ui/react';
 import { useRouter } from 'next/router';
 import { useLoading } from '@/web/common/hooks/useLoading';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { throttle } from 'lodash';
 import Auth from './auth';
 import Navbar from './navbar';
 import NavbarPhone from './navbarPhone';
 import { useQuery } from '@tanstack/react-query';
-import { useUserStore } from '@/web/support/store/user';
-import { getUnreadCount } from '@/web/support/api/user';
+import { useUserStore } from '@/web/support/user/useUserStore';
+import { getUnreadCount } from '@/web/support/user/api';
 
 const pcUnShowLayoutRoute: Record<string, boolean> = {
   '/': true,
@@ -30,7 +30,7 @@ const Layout = ({ children }: { children: JSX.Element }) => {
   const router = useRouter();
   const { colorMode, setColorMode } = useColorMode();
   const { Loading } = useLoading();
-  const { loading, setScreenWidth, isPc, loadGitStar } = useGlobalStore();
+  const { loading, setScreenWidth, isPc, loadGitStar } = useSystemStore();
   const { userInfo } = useUserStore();
 
   const isChatPage = useMemo(

projects/app/src/components/Layout/navbar.tsx

@@ -1,16 +1,16 @@
 import React, { useMemo } from 'react';
 import { Box, Flex, Link } from '@chakra-ui/react';
 import { useRouter } from 'next/router';
-import { useUserStore } from '@/web/support/store/user';
-import { useChatStore } from '@/web/core/store/chat';
+import { useUserStore } from '@/web/support/user/useUserStore';
+import { useChatStore } from '@/web/core/chat/storeChat';
 import { HUMAN_ICON } from '@/constants/chat';
-import { feConfigs } from '@/web/common/store/static';
+import { feConfigs } from '@/web/common/system/staticData';
 import NextLink from 'next/link';
 import Badge from '../Badge';
 import Avatar from '../Avatar';
 import MyIcon from '../Icon';
 import { useTranslation } from 'next-i18next';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import MyTooltip from '../MyTooltip';
 
 export enum NavbarTypeEnum {
@@ -22,7 +22,7 @@ const Navbar = ({ unread }: { unread: number }) => {
   const { t } = useTranslation();
   const router = useRouter();
   const { userInfo } = useUserStore();
-  const { gitStar } = useGlobalStore();
+  const { gitStar } = useSystemStore();
   const { lastChatAppId, lastChatId } = useChatStore();
   const navbarList = useMemo(
     () => [
@@ -44,8 +44,8 @@ const Navbar = ({ unread }: { unread: number }) => {
         label: t('navbar.Datasets'),
         icon: 'dbLight',
         activeIcon: 'dbFill',
-        link: `/kb/list`,
-        activeLink: ['/kb/list', '/kb/detail']
+        link: `/dataset/list`,
+        activeLink: ['/dataset/list', '/dataset/detail']
       },
       ...(feConfigs?.show_appStore
         ? [

projects/app/src/components/Layout/navbarPhone.tsx

@@ -1,7 +1,7 @@
 import React, { useMemo } from 'react';
 import { useRouter } from 'next/router';
 import { Flex, Box } from '@chakra-ui/react';
-import { useChatStore } from '@/web/core/store/chat';
+import { useChatStore } from '@/web/core/chat/storeChat';
 import { useTranslation } from 'react-i18next';
 import Badge from '../Badge';
 import MyIcon from '../Icon';

projects/app/src/components/Markdown/chat/Quote.tsx

@@ -1,8 +1,8 @@
 import React, { useMemo } from 'react';
 import { Box, useTheme } from '@chakra-ui/react';
-import { getFileAndOpen } from '@/web/common/utils/file';
+import { getFileAndOpen } from '@/web/common/file/utils';
 import { useToast } from '@/web/common/hooks/useToast';
-import { getErrText } from '@/utils/tools';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 
 type QuoteItemType = {
   file_id?: string;

projects/app/src/components/MyModal/index.tsx

@@ -22,7 +22,7 @@ const MyModal = ({
   title,
   children,
   isCentered,
-  w = 'auto',
+  w = '100%',
   maxW = ['90vw', '600px'],
   ...props
 }: Props) => {
@@ -44,7 +44,12 @@ const MyModal = ({
       >
         {!!title && <ModalHeader>{title}</ModalHeader>}
         {onClose && <ModalCloseButton />}
-        <Box overflow={'overlay'} h={'100%'} display={'flex'} flexDirection={'column'}>
+        <Box
+          overflow={props.overflow || 'overlay'}
+          h={'100%'}
+          display={'flex'}
+          flexDirection={'column'}
+        >
           {children}
         </Box>
       </ModalContent>

projects/app/src/components/MyTooltip/index.tsx

@@ -1,13 +1,13 @@
 import React from 'react';
 import { Tooltip, TooltipProps } from '@chakra-ui/react';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 
 interface Props extends TooltipProps {
   forceShow?: boolean;
 }
 
 const MyTooltip = ({ children, forceShow = false, shouldWrapChildren = true, ...props }: Props) => {
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   return isPc || forceShow ? (
     <Tooltip
       bg={'white'}

projects/app/src/components/PromptTemplate/index.tsx

@@ -1,7 +1,7 @@
 import React, { useState } from 'react';
 import MyModal from '../MyModal';
 import { Box, Button, Grid, useTheme } from '@chakra-ui/react';
-import { PromptTemplateItem } from '@fastgpt/core/ai/type';
+import { PromptTemplateItem } from '@fastgpt/global/core/ai/type.d';
 import { ModalBody, ModalFooter } from '@chakra-ui/react';
 
 const PromptTemplate = ({

projects/app/src/components/common/ParentPaths/index.tsx

@@ -0,0 +1,62 @@
+import MyIcon from '@/components/Icon';
+import { Box, Flex } from '@chakra-ui/react';
+import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
+import React, { useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+
+const ParentPaths = (props: {
+  paths?: ParentTreePathItemType[];
+  rootName?: string;
+  FirstPathDom?: React.ReactNode;
+  onClick: (parentId: string) => void;
+}) => {
+  const { t } = useTranslation();
+  const { paths = [], rootName = t('common.folder.Root Path'), FirstPathDom, onClick } = props;
+  const concatPaths = useMemo(
+    () => [
+      {
+        parentId: '',
+        parentName: rootName
+      },
+      ...paths
+    ],
+    [rootName, paths]
+  );
+
+  return paths.length === 0 && !!FirstPathDom ? (
+    <>{FirstPathDom}</>
+  ) : (
+    <Flex flex={1}>
+      {concatPaths.map((item, i) => (
+        <Flex key={item.parentId} alignItems={'center'}>
+          <Box
+            fontSize={['md', 'lg']}
+            py={1}
+            px={[0, 2]}
+            borderRadius={'md'}
+            {...(i === concatPaths.length - 1
+              ? {
+                  cursor: 'default'
+                }
+              : {
+                  cursor: 'pointer',
+                  _hover: {
+                    bg: 'myGray.100'
+                  },
+                  onClick: () => {
+                    onClick(item.parentId);
+                  }
+                })}
+          >
+            {item.parentName}
+          </Box>
+          {i !== concatPaths.length - 1 && (
+            <MyIcon name={'rightArrowLight'} color={'myGray.500'} w={['18px', '24px']} />
+          )}
+        </Flex>
+      ))}
+    </Flex>
+  );
+};
+
+export default React.memo(ParentPaths);

projects/app/src/components/core/dataset/SelectModal.tsx

@@ -1,9 +1,9 @@
-import { getDatasets, getDatasetPaths } from '@/web/core/api/dataset';
+import { getDatasets, getDatasetPaths } from '@/web/core/dataset/api';
 import MyModal from '@/components/MyModal';
 import { useQuery } from '@tanstack/react-query';
 import React, { Dispatch, useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { Box, Flex, ModalHeader } from '@chakra-ui/react';
 import MyIcon from '@/components/Icon';
 
@@ -30,7 +30,7 @@ const DatasetSelectContainer = ({
   children: React.ReactNode;
 }) => {
   const { t } = useTranslation();
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
 
   return (
     <MyModal isOpen={isOpen} onClose={onClose} w={'100%'} maxW={['90vw', '900px']} isCentered>
@@ -86,11 +86,11 @@ const DatasetSelectContainer = ({
   );
 };
 
-export const useDatasetSelect = () => {
+export function useDatasetSelect() {
   const { t } = useTranslation();
   const [parentId, setParentId] = useState<string>();
 
-  const { data } = useQuery(['loadDatasetData', parentId], () =>
+  const { data, isLoading } = useQuery(['loadDatasetData', parentId], () =>
     Promise.all([getDatasets({ parentId }), getDatasetPaths(parentId)])
   );
 
@@ -98,7 +98,7 @@ export const useDatasetSelect = () => {
     () => [
       {
         parentId: '',
-        parentName: t('kb.My Dataset')
+        parentName: t('dataset.My Dataset')
       },
       ...(data?.[1] || [])
     ],
@@ -109,8 +109,9 @@ export const useDatasetSelect = () => {
     parentId,
     setParentId,
     datasets: data?.[0] || [],
-    paths
+    paths,
+    isLoading
   };
-};
+}
 
 export default DatasetSelectContainer;

projects/app/src/components/support/apikey/Table.tsx

@@ -25,14 +25,14 @@ import {
   createAOpenApiKey,
   delOpenApiById,
   putOpenApiKey
-} from '@/web/support/api/openapi';
+} from '@/web/support/openapi/api';
 import type { EditApiKeyProps } from '@/global/support/api/openapiReq';
 import { useQuery, useMutation } from '@tanstack/react-query';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import dayjs from 'dayjs';
 import { AddIcon, QuestionOutlineIcon } from '@chakra-ui/icons';
 import { useCopyData } from '@/web/common/hooks/useCopyData';
-import { feConfigs } from '@/web/common/store/static';
+import { feConfigs } from '@/web/common/system/staticData';
 import { useTranslation } from 'react-i18next';
 import MyIcon from '@/components/Icon';
 import MyModal from '@/components/MyModal';

projects/app/src/constants/flow/ModuleTemplate.ts

@@ -9,7 +9,7 @@ import {
 } from './index';
 import type { AppItemType } from '@/types/app';
 import type { FlowModuleTemplateType } from '@/types/core/app/flow';
-import { chatModelList, cqModelList } from '@/web/common/store/static';
+import { chatModelList, cqModelList } from '@/web/common/system/staticData';
 import {
   Input_Template_History,
   Input_Template_TFSwitch,
@@ -240,7 +240,7 @@ export const ChatModule: FlowModuleTemplateType = {
 };
 
 export const KBSearchModule: FlowModuleTemplateType = {
-  flowType: FlowModuleTypeEnum.kbSearchNode,
+  flowType: FlowModuleTypeEnum.datasetSearchNode,
   logo: '/imgs/module/db.png',
   name: '知识库搜索',
   intro: '去知识库中搜索对应的答案。可作为 AI 对话引用参考。',
@@ -248,7 +248,7 @@ export const KBSearchModule: FlowModuleTemplateType = {
   inputs: [
     Input_Template_TFSwitch,
     {
-      key: 'kbList',
+      key: 'datasets',
       type: FlowInputItemTypeEnum.selectDataset,
       label: '关联的知识库',
       value: [],
@@ -900,7 +900,7 @@ export const appTemplates: (AppItemType & {
       {
         moduleId: 'kbSearch',
         name: '知识库搜索',
-        flowType: 'kbSearchNode',
+        flowType: 'datasetSearchNode',
         showStatus: true,
         position: {
           x: 956.0838440206068,
@@ -908,7 +908,7 @@ export const appTemplates: (AppItemType & {
         },
         inputs: [
           {
-            key: 'kbList',
+            key: 'datasets',
             type: 'custom',
             label: '关联的知识库',
             value: [],
@@ -1952,7 +1952,7 @@ export const appTemplates: (AppItemType & {
       {
         moduleId: 'fljhzy',
         name: '知识库搜索',
-        flowType: 'kbSearchNode',
+        flowType: 'datasetSearchNode',
         showStatus: true,
         position: {
           x: 1305.5374262228029,
@@ -1960,7 +1960,7 @@ export const appTemplates: (AppItemType & {
         },
         inputs: [
           {
-            key: 'kbList',
+            key: 'datasets',
             type: 'custom',
             label: '关联的知识库',
             value: [],

projects/app/src/constants/flow/index.ts

@@ -34,7 +34,7 @@ export enum FlowModuleTypeEnum {
   questionInput = 'questionInput',
   historyNode = 'historyNode',
   chatNode = 'chatNode',
-  kbSearchNode = 'kbSearchNode',
+  datasetSearchNode = 'datasetSearchNode',
   tfSwitchNode = 'tfSwitchNode',
   answerNode = 'answerNode',
   classifyQuestion = 'classifyQuestion',

projects/app/src/constants/model.ts

@@ -1,5 +1,5 @@
 import type { AppSchema } from '@/types/mongoSchema';
-import type { OutLinkEditType } from '@fastgpt/support/outLink/type.d';
+import type { OutLinkEditType } from '@fastgpt/global/support/outLink/type.d';
 import type {
   LLMModelItemType,
   ChatModelItemType,

projects/app/src/constants/plugin.ts

@@ -1,10 +1 @@
-export enum TrainingModeEnum {
-  'qa' = 'qa',
-  'index' = 'index'
-}
-export const TrainingTypeMap = {
-  [TrainingModeEnum.qa]: 'qa',
-  [TrainingModeEnum.index]: 'index'
-};
-
 export const PgDatasetTableName = 'modeldata';

projects/app/src/global/common/api/billReq.d.ts

@@ -1,3 +0,0 @@
-export type CreateTrainingBillType = {
-  name: string;
-};

projects/app/src/global/common/api/pluginRes.d.ts

@@ -1,4 +0,0 @@
-export type FetchResultItem = {
-  url: string;
-  content: string;
-};

projects/app/src/global/common/api/systemRes.d.ts

@@ -4,7 +4,7 @@ import type {
   LLMModelItemType,
   VectorModelItemType
 } from '@/types/model';
-import type { FeConfigsType } from '@fastgpt/common/type/index.d';
+import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d';
 
 export type InitDateResponse = {
   chatModels: ChatModelItemType[];

projects/app/src/global/common/string/tools.ts

@@ -1,5 +1,18 @@
-import { getErrText } from './tools';
-import { countPromptTokens } from './common/tiktoken';
+import { getErrText } from '@fastgpt/global/common/error/utils';
+import { countPromptTokens } from '@/global/common/tiktoken';
+
+/* 
+    replace {{variable}} to value
+*/
+export function replaceVariable(text: string, obj: Record<string, string | number>) {
+  for (const key in obj) {
+    const val = obj[key];
+    if (typeof val !== 'string') continue;
+
+    text = text.replace(new RegExp(`{{(${key})}}`, 'g'), val);
+  }
+  return text || '';
+}
 
 /**
  * text split into chunks

projects/app/src/global/common/tiktoken/index.ts

@@ -1,8 +1,8 @@
 /* Only the token of gpt-3.5-turbo is used */
 import { ChatItemType } from '@/types/chat';
 import { Tiktoken } from 'js-tiktoken/lite';
-import { adaptChat2GptMessages } from '../adapt/message';
-import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/core/ai/constant';
+import { adaptChat2GptMessages } from '@/utils/common/adapt/message';
+import { ChatCompletionRequestMessageRoleEnum } from '@fastgpt/global/core/ai/constant';
 import encodingJson from './cl100k_base.json';
 
 /* init tikToken obj */
@@ -27,7 +27,10 @@ export function getTikTokenEnc() {
 }
 
 /* count one prompt tokens */
-export function countPromptTokens(prompt = '', role: `${ChatCompletionRequestMessageRoleEnum}`) {
+export function countPromptTokens(
+  prompt = '',
+  role: '' | `${ChatCompletionRequestMessageRoleEnum}` = ''
+) {
   const enc = getTikTokenEnc();
   const text = `${role}\n${prompt}`;
   try {

projects/app/src/global/core/api/aiReq.d.ts

@@ -1,4 +1,4 @@
-import { ChatCompletionRequestMessage } from '@fastgpt/core/ai/type';
+import { ChatCompletionRequestMessage } from '@fastgpt/global/core/ai/type.d';
 
 export type CreateQuestionGuideParams = {
   messages: ChatCompletionRequestMessage[];

projects/app/src/global/core/api/chatReq.d.ts

@@ -1,6 +1,5 @@
-export type AdminUpdateFeedbackParams = {
+import { MarkDataType } from '../dataset/type';
+
+export type AdminUpdateFeedbackParams = MarkDataType & {
   chatItemId: string;
-  kbId: string;
-  dataId: string;
-  content: string;
 };

projects/app/src/global/core/api/datasetReq.d.ts

@@ -1,8 +1,9 @@
-import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
+import { DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import type { RequestPaging } from '@/types';
-import { TrainingModeEnum } from '@/constants/plugin';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
 import type { SearchTestItemType } from '@/types/core/dataset';
-import { DatasetDataItemType } from '@/types/core/dataset/data';
+import { DatasetChunkItemType, UploadChunkItemType } from '@fastgpt/global/core/dataset/type';
+import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type';
 
 /* ===== dataset ===== */
 export type DatasetUpdateParams = {
@@ -22,38 +23,50 @@ export type CreateDatasetParams = {
 };
 
 export type SearchTestProps = {
-  kbId: string;
+  datasetId: string;
   text: string;
 };
 
-/* ======= file =========== */
-export type GetFileListProps = RequestPaging & {
-  kbId: string;
-  searchText: string;
+/* ======= collections =========== */
+export type GetDatasetCollectionsProps = RequestPaging & {
+  datasetId: string;
+  parentId?: string;
+  searchText?: string;
+  simple?: boolean;
+  selectFolder?: boolean;
+};
+export type CreateDatasetCollectionParams = {
+  datasetId: string;
+  parentId?: string;
+  name: string;
+  type: `${DatasetCollectionTypeEnum}`;
+  metadata?: DatasetCollectionSchemaType['metadata'];
+  updateTime?: string;
+};
+export type UpdateDatasetCollectionParams = {
+  id: string;
+  parentId?: string;
+  name?: string;
+  metadata?: DatasetCollectionSchemaType['metadata'];
 };
 
-export type UpdateFileProps = { id: string; name?: string; datasetUsed?: boolean };
-
-export type MarkFileUsedProps = { fileIds: string[] };
-
 /* ==== data ===== */
+export type SetOneDatasetDataProps = {
+  id?: string;
+  datasetId: string;
+  collectionId: string;
+  q?: string; // embedding content
+  a?: string; // bonus content
+};
 export type PushDataProps = {
-  kbId: string;
-  data: DatasetDataItemType[];
+  collectionId: string;
+  data: DatasetChunkItemType[];
   mode: `${TrainingModeEnum}`;
   prompt?: string;
   billId?: string;
 };
 
-export type UpdateDatasetDataPrams = {
-  dataId: string;
-  kbId: string;
-  a?: string;
-  q?: string;
-};
-
 export type GetDatasetDataListProps = RequestPaging & {
-  kbId: string;
-  searchText: string;
-  fileId: string;
+  searchText?: string;
+  collectionId: string;
 };

projects/app/src/global/core/api/datasetRes.d.ts

@@ -2,11 +2,11 @@ import type { RequestPaging } from '@/types';
 import { TrainingModeEnum } from '@/constants/plugin';
 import type { SearchTestItemType } from '@/types/core/dataset';
 import { DatasetDataItemType } from '@/types/core/dataset/data';
+import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type';
 
 /* ===== dataset ===== */
-export type SearchTestResponseType = SearchTestItemType['results'];
 
-/* ======= file =========== */
+/* ======= collection =========== */
 
 /* ==== data ===== */
 export type PushDataResponse = {

projects/app/src/global/core/dataset/request.d.ts

@@ -0,0 +1,5 @@
+/* ================= dataset ===================== */
+
+/* ================= collection ===================== */
+
+/* ================= data ===================== */

projects/app/src/global/core/dataset/response.d.ts

@@ -0,0 +1,23 @@
+import { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder/type';
+import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d';
+
+/* ================= dataset ===================== */
+
+/* ================= collection ===================== */
+export type DatasetCollectionsListItemType = {
+  _id: string;
+  parentId?: string;
+  name: string;
+  type: DatasetCollectionSchemaType['type'];
+  updateTime: Date;
+  dataAmount?: number;
+  trainingAmount: number;
+  metadata: DatasetCollectionSchemaType['metadata'];
+};
+
+/* ================= data ===================== */
+export type DatasetDataListItemType = {
+  id: string;
+  q: string; // embedding content
+  a: string; // bonus content
+};

projects/app/src/global/core/dataset/type.d.ts

@@ -0,0 +1,7 @@
+export type MarkDataType = {
+  dataId: string;
+  datasetId: string;
+  collectionId: string;
+  q: string;
+  a?: string;
+};

projects/app/src/global/core/prompt/AIChat.ts

@@ -1,4 +1,4 @@
-import { PromptTemplateItem } from '@fastgpt/core/ai/type.d';
+import { PromptTemplateItem } from '@fastgpt/global/core/ai/type.d';
 
 export const Prompt_QuoteTemplateList: PromptTemplateItem[] = [
   {
@@ -9,7 +9,7 @@ export const Prompt_QuoteTemplateList: PromptTemplateItem[] = [
   {
     title: '全部变量',
     desc: '包含 q 和 a 两个变量的标准模板',
-    value: `{instruction:"{{q}}",output:"{{a}}",source:"{{source}}",file_id:"{{file_id}}",index:"{{index}}"}`
+    value: `{instruction:"{{q}}",output:"{{a}}",source:"{{source}}",sourceId:"{{sourceId}}",index:"{{index}}"}`
   }
 ];
 
@@ -24,7 +24,7 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [
 对话要求：
 1. 背景知识是最新的，其中 instruction 是相关介绍，output 是预期回答或补充。
 2. 使用背景知识回答问题。
-3. 背景知识无法满足问题时，你需严谨的回答问题。
+3. 使用对话的风格回答我的问题，答案要和背景知识表述一致。
 我的问题是:"{{question}}"`
   },
   {

projects/app/src/global/support/api/openapiReq.d.ts

@@ -1,4 +1,4 @@
-import type { OpenApiSchema } from '@fastgpt/support/openapi/type.d';
+import type { OpenApiSchema } from '@fastgpt/global/support/openapi/type';
 
 export type GetApiKeyProps = {
   appId?: string;

projects/app/src/pages/_app.tsx

@@ -8,12 +8,12 @@ import { theme } from '@/web/styles/theme';
 import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import NProgress from 'nprogress'; //nprogress module
 import Router from 'next/router';
-import { clientInitData, feConfigs } from '@/web/common/store/static';
+import { clientInitData, feConfigs } from '@/web/common/system/staticData';
 import { appWithTranslation, useTranslation } from 'next-i18next';
 import { getLangStore, setLangStore } from '@/web/common/utils/i18n';
 import { useRouter } from 'next/router';
-import { useGlobalStore } from '@/web/common/store/global';
-import type { FeConfigsType } from '@fastgpt/common/type/index.d';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
+import type { FeConfigsType } from '@fastgpt/global/common/system/types/index.d';
 
 import 'nprogress/nprogress.css';
 import '@/web/styles/reset.scss';
@@ -38,7 +38,7 @@ function App({ Component, pageProps }: AppProps) {
   const router = useRouter();
   const { hiId } = router.query as { hiId?: string };
   const { i18n } = useTranslation();
-  const { setLastRoute } = useGlobalStore();
+  const { setLastRoute } = useSystemStore();
   const [scripts, setScripts] = useState<FeConfigsType['scripts']>([]);
 
   useEffect(() => {

projects/app/src/pages/_error.tsx

@@ -1,11 +1,11 @@
 import { useEffect } from 'react';
 import { useRouter } from 'next/router';
 import { serviceSideProps } from '@/web/common/utils/i18n';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 
 function Error() {
   const router = useRouter();
-  const { lastRoute } = useGlobalStore();
+  const { lastRoute } = useSystemStore();
 
   useEffect(() => {
     setTimeout(() => {

projects/app/src/pages/account/components/BillDetail.tsx

@@ -14,7 +14,7 @@ import {
 import { UserBillType } from '@/types/user';
 import dayjs from 'dayjs';
 import { BillSourceMap } from '@/constants/user';
-import { formatPrice } from '@fastgpt/common/bill/index';
+import { formatPrice } from '@fastgpt/global/common/bill/tools';
 import MyModal from '@/components/MyModal';
 import { useTranslation } from 'react-i18next';
 

projects/app/src/pages/account/components/BillTable.tsx

@@ -12,7 +12,7 @@ import {
   Button
 } from '@chakra-ui/react';
 import { BillSourceMap } from '@/constants/user';
-import { getUserBills } from '@/web/common/api/bill';
+import { getUserBills } from '@/web/common/bill/api';
 import type { UserBillType } from '@/types/user';
 import { usePagination } from '@/web/common/hooks/usePagination';
 import { useLoading } from '@/web/common/hooks/useLoading';
@@ -21,7 +21,7 @@ import MyIcon from '@/components/Icon';
 import DateRangePicker, { type DateRangeType } from '@/components/DateRangePicker';
 import { addDays } from 'date-fns';
 import dynamic from 'next/dynamic';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { useTranslation } from 'next-i18next';
 const BillDetail = dynamic(() => import('./BillDetail'));
 
@@ -32,7 +32,7 @@ const BillTable = () => {
     from: addDays(new Date(), -7),
     to: new Date()
   });
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
 
   const {
     data: bills,

projects/app/src/pages/account/components/Info.tsx

@@ -15,15 +15,15 @@ import {
 import { useForm } from 'react-hook-form';
 import { UserUpdateParams } from '@/types/user';
 import { useToast } from '@/web/common/hooks/useToast';
-import { useUserStore } from '@/web/support/store/user';
+import { useUserStore } from '@/web/support/user/useUserStore';
 import { UserType } from '@/types/user';
 import { useQuery } from '@tanstack/react-query';
 import dynamic from 'next/dynamic';
-import { useSelectFile } from '@/web/common/hooks/useSelectFile';
-import { compressImg } from '@/web/common/utils/file';
-import { feConfigs, systemVersion } from '@/web/common/store/static';
+import { useSelectFile } from '@/web/common/file/hooks/useSelectFile';
+import { compressImg } from '@/web/common/file/utils';
+import { feConfigs, systemVersion } from '@/web/common/system/staticData';
 import { useTranslation } from 'next-i18next';
-import { timezoneList } from '@/utils/user';
+import { timezoneList } from '@fastgpt/global/common/time/timezone';
 import Loading from '@/components/Loading';
 import Avatar from '@/components/Avatar';
 import MyIcon from '@/components/Icon';

projects/app/src/pages/account/components/InformTable.tsx

@@ -1,17 +1,17 @@
 import React from 'react';
 import { Box, Flex, useTheme } from '@chakra-ui/react';
-import { getInforms, readInform } from '@/web/support/api/user';
+import { getInforms, readInform } from '@/web/support/user/api';
 import { usePagination } from '@/web/common/hooks/usePagination';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import type { informSchema } from '@/types/mongoSchema';
 import { formatTimeToChatTime } from '@/utils/tools';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import MyIcon from '@/components/Icon';
 
 const BillTable = () => {
   const theme = useTheme();
   const { Loading } = useLoading();
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   const {
     data: informs,
     isLoading,

projects/app/src/pages/account/components/PayModal.tsx

@@ -1,14 +1,14 @@
 import React, { useState, useCallback } from 'react';
 import { ModalFooter, ModalBody, Button, Input, Box, Grid } from '@chakra-ui/react';
-import { getPayCode, checkPayResult } from '@/web/common/api/bill';
+import { getPayCode, checkPayResult } from '@/web/common/bill/api';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useQuery } from '@tanstack/react-query';
 import { useRouter } from 'next/router';
-import { getErrText } from '@/utils/tools';
+import { getErrText } from '@fastgpt/global/common/error/utils';
 import { useTranslation } from 'react-i18next';
 import Markdown from '@/components/Markdown';
 import MyModal from '@/components/MyModal';
-import { priceMd } from '@/web/common/store/static';
+import { priceMd } from '@/web/common/system/staticData';
 
 const PayModal = ({ onClose }: { onClose: () => void }) => {
   const router = useRouter();
@@ -69,12 +69,7 @@ const PayModal = ({ onClose }: { onClose: () => void }) => {
       title={t('user.Pay')}
       isCentered={!payId}
     >
-      <ModalBody
-        p={0}
-        h={payId ? 'auto' : ['auto', '70vh']}
-        display={'flex'}
-        flexDirection={'column'}
-      >
+      <ModalBody p={0} minH={payId ? 'auto' : '70vh'} display={'flex'} flexDirection={'column'}>
         {!payId && (
           <>
             <Grid gridTemplateColumns={'repeat(4,1fr)'} gridGap={5} mb={4} px={6}>

projects/app/src/pages/account/components/PayRecordTable.tsx

@@ -11,11 +11,11 @@ import {
   Flex,
   Box
 } from '@chakra-ui/react';
-import { getPayOrders, checkPayResult } from '@/web/common/api/bill';
+import { getPayOrders, checkPayResult } from '@/web/common/bill/api';
 import { PaySchema } from '@/types/mongoSchema';
 import dayjs from 'dayjs';
 import { useQuery } from '@tanstack/react-query';
-import { formatPrice } from '@fastgpt/common/bill/index';
+import { formatPrice } from '@fastgpt/global/common/bill/tools';
 import { useToast } from '@/web/common/hooks/useToast';
 import { useLoading } from '@/web/common/hooks/useLoading';
 import MyIcon from '@/components/Icon';
@@ -58,7 +58,7 @@ const PayRecordTable = () => {
   );
 
   return (
-    <Box position={'relative'} h={'100%'}>
+    <Box position={'relative'} h={'100%'} overflow={'overlay'}>
       {!isInitialLoading && payOrders.length === 0 ? (
         <Flex h={'100%'} flexDirection={'column'} alignItems={'center'} justifyContent={'center'}>
           <MyIcon name="empty" w={'48px'} h={'48px'} color={'transparent'} />
@@ -67,7 +67,7 @@ const PayRecordTable = () => {
           </Box>
         </Flex>
       ) : (
-        <TableContainer py={[0, 5]} px={[3, 8]} h={'100%'} overflow={'overlay'}>
+        <TableContainer py={[0, 5]} px={[3, 8]}>
           <Table>
             <Thead>
               <Tr>

projects/app/src/pages/account/components/Promotion.tsx

@@ -16,8 +16,8 @@ import {
 } from '@chakra-ui/react';
 import { useTranslation } from 'next-i18next';
 import { useQuery } from '@tanstack/react-query';
-import { getPromotionInitData, getPromotionRecords } from '@/web/support/api/user';
-import { useUserStore } from '@/web/support/store/user';
+import { getPromotionInitData, getPromotionRecords } from '@/web/support/user/api';
+import { useUserStore } from '@/web/support/user/useUserStore';
 import { useLoading } from '@/web/common/hooks/useLoading';
 
 import MyTooltip from '@/components/MyTooltip';

projects/app/src/pages/account/components/UpdatePswModal.tsx

@@ -4,7 +4,7 @@ import MyModal from '@/components/MyModal';
 import { useTranslation } from 'react-i18next';
 import { useForm } from 'react-hook-form';
 import { useRequest } from '@/web/common/hooks/useRequest';
-import { updatePasswordByOld } from '@/web/support/api/user';
+import { updatePasswordByOld } from '@/web/support/user/api';
 
 type FormType = {
   oldPsw: string;

projects/app/src/pages/account/index.tsx

@@ -1,17 +1,17 @@
 import React, { useCallback, useRef } from 'react';
 import { Box, Flex, useTheme } from '@chakra-ui/react';
-import { useGlobalStore } from '@/web/common/store/global';
+import { useSystemStore } from '@/web/common/system/useSystemStore';
 import { useRouter } from 'next/router';
 import dynamic from 'next/dynamic';
-import { clearToken } from '@/utils/user';
-import { useUserStore } from '@/web/support/store/user';
+import { clearToken } from '@/web/support/user/auth';
+import { useUserStore } from '@/web/support/user/useUserStore';
 import { useConfirm } from '@/web/common/hooks/useConfirm';
 import PageContainer from '@/components/PageContainer';
 import SideTabs from '@/components/SideTabs';
 import Tabs from '@/components/Tabs';
 import UserInfo from './components/Info';
 import { serviceSideProps } from '@/web/common/utils/i18n';
-import { feConfigs } from '@/web/common/store/static';
+import { feConfigs } from '@/web/common/system/staticData';
 import { useTranslation } from 'react-i18next';
 import Script from 'next/script';
 
@@ -86,7 +86,7 @@ const Account = ({ currentTab }: { currentTab: `${TabEnum}` }) => {
 
   const router = useRouter();
   const theme = useTheme();
-  const { isPc } = useGlobalStore();
+  const { isPc } = useSystemStore();
   const { setUserInfo } = useUserStore();
 
   const setCurrentTab = useCallback(

projects/app/src/pages/api/admin/initv43.ts

@@ -1,35 +0,0 @@
-// Next.js API route support: https://nextjs.org/docs/api-routes/introduction
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
-import { PgClient } from '@/service/pg';
-import { PgDatasetTableName } from '@/constants/plugin';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    await authUser({ req, authRoot: true });
-
-    const { rowCount } = await PgClient.query(`SELECT 1
-    FROM   information_schema.columns 
-    WHERE  table_schema = 'public'
-    AND    table_name   = '${PgDatasetTableName}'
-    AND    column_name  = 'file_id'`);
-
-    if (rowCount > 0) {
-      return jsonRes(res, {
-        data: '已经存在file_id字段'
-      });
-    }
-
-    jsonRes(res, {
-      data: await PgClient.query(
-        `ALTER TABLE ${PgDatasetTableName} ADD COLUMN file_id VARCHAR(100)`
-      )
-    });
-  } catch (error) {
-    jsonRes(res, {
-      code: 500,
-      error
-    });
-  }
-}

projects/app/src/pages/api/admin/initv44.ts

@@ -1,10 +1,10 @@
 // Next.js API route support: https://nextjs.org/docs/api-routes/introduction
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { DatasetTypeEnum } from '@fastgpt/core/dataset/constant';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 

projects/app/src/pages/api/admin/initv441.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase } from '@/service/mongo';
-import mongoose from '@fastgpt/common/mongo';
+import mongoose from '@fastgpt/service/common/mongo';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 

projects/app/src/pages/api/admin/initv442.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase, Bill } from '@/service/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/admin/initv445.ts

@@ -1,6 +1,6 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase, App } from '@/service/mongo';
 import { FlowInputItemTypeEnum, FlowModuleTypeEnum } from '@/constants/flow';
 import { SystemInputEnum } from '@/constants/app';

projects/app/src/pages/api/admin/initv447.ts

@@ -1,11 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { connectToDatabase } from '@/service/mongo';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
-import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
-import { Types, connectionMongo } from '@fastgpt/common/mongo';
+import { DatasetSpecialIdEnum } from '@fastgpt/global/core/dataset/constant';
+import { Types, connectionMongo } from '@fastgpt/service/common/mongo';
 import { delay } from '@/utils/tools';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {

projects/app/src/pages/api/admin/initv451.ts

@@ -0,0 +1,344 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { App, connectToDatabase } from '@/service/mongo';
+import { PgClient } from '@/service/pg';
+import { connectionMongo } from '@fastgpt/service/common/mongo';
+import { PgDatasetTableName } from '@/constants/plugin';
+import { FlowModuleTypeEnum } from '@/constants/flow';
+import { delay } from '@/utils/tools';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { strIsLink } from '@fastgpt/global/common/string/tools';
+import { GridFSStorage } from '@/service/lib/gridfs';
+import { Types } from 'mongoose';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse) {
+  try {
+    const { limit = 50 } = req.body as { limit: number };
+    await connectToDatabase();
+
+    console.log('rename');
+    await rename();
+
+    console.log('init mongo data');
+    await initMongo(limit);
+
+    console.log('create collection');
+    await createCollection();
+
+    console.log('update pg collectionId');
+    await updatePgCollection();
+    console.log('init done');
+
+    jsonRes(res, {
+      data: {}
+    });
+  } catch (error) {
+    console.log(error);
+
+    jsonRes(res, {
+      code: 500,
+      error
+    });
+  }
+}
+
+async function rename() {
+  // rename mongo kbs -> datasets
+  try {
+    const collections = await connectionMongo.connection.db
+      .listCollections({ name: 'kbs' })
+      .toArray();
+    if (collections.length > 0) {
+      const kbCollection = connectionMongo.connection.db.collection('kbs');
+      await kbCollection.rename('datasets', { dropTarget: true });
+      console.log('success rename kbs -> datasets');
+    }
+  } catch (error) {
+    console.log('error： rename kbs -> datasets', error);
+  }
+
+  // rename pg: kb_id -> dataset_id
+  try {
+    const { rows } = await PgClient.query(`SELECT EXISTS (
+      SELECT 1
+      FROM information_schema.columns
+      WHERE table_name = '${PgDatasetTableName}'
+      AND column_name = 'kb_id'
+  );`);
+
+    if (rows[0].exists) {
+      await PgClient.query(`ALTER TABLE ${PgDatasetTableName} RENAME COLUMN kb_id TO dataset_id`);
+      console.log('success rename kb_id -> dataset_id');
+    }
+  } catch (error) {
+    console.log('error： rename kb_id -> dataset_id', error);
+  }
+  // rename pg: file_id -> collection_id
+  try {
+    const { rows } = await PgClient.query(`SELECT EXISTS (
+      SELECT 1
+      FROM information_schema.columns
+      WHERE table_name = '${PgDatasetTableName}'
+      AND column_name = 'file_id'
+  );`);
+
+    if (rows[0].exists) {
+      await PgClient.query(
+        `ALTER TABLE ${PgDatasetTableName} RENAME COLUMN file_id TO collection_id`
+      );
+      console.log('success rename file_id -> collection_id');
+    }
+  } catch (error) {
+    console.log('error： rename file_id -> collection_id', error);
+  }
+}
+
+async function initMongo(limit: number) {
+  let success = 0;
+
+  async function initApp(limit = 100): Promise<any> {
+    // 遍历所有 app，更新 app modules 里的 FlowModuleTypeEnum.kbSearchNode
+    const apps = await App.find({ inited: false }).limit(limit);
+
+    if (apps.length === 0) return;
+
+    try {
+      await Promise.all(
+        apps.map(async (app) => {
+          const modules = app.toObject().modules;
+          // @ts-ignore
+          app.inited = true;
+
+          modules.forEach((module) => {
+            // @ts-ignore
+            if (module.flowType === 'kbSearchNode') {
+              module.flowType = FlowModuleTypeEnum.datasetSearchNode;
+              module.inputs.forEach((input) => {
+                if (input.key === 'kbList') {
+                  input.key = 'datasets';
+                  input.value?.forEach((item: any) => {
+                    item.datasetId = item.kbId;
+                  });
+                }
+              });
+            }
+          });
+
+          app.modules = JSON.parse(JSON.stringify(modules));
+          await app.save();
+        })
+      );
+      success += limit;
+      console.log('mongo init:', success);
+      return initApp(limit);
+    } catch (error) {
+      return initApp(limit);
+    }
+  }
+
+  // init app
+  await App.updateMany(
+    {},
+    {
+      $set: {
+        inited: false
+      }
+    }
+  );
+
+  const totalApp = await App.countDocuments();
+  console.log(`total app: ${totalApp}`);
+  await delay(2000);
+  console.log('start init app');
+  await initApp(limit);
+  console.log('init mongo success');
+}
+
+type RowType = { user_id: string; dataset_id: string; collection_id: string };
+async function createCollection() {
+  let success = 0;
+
+  const { rows, rowCount } = await PgClient.query(`SELECT user_id,dataset_id,collection_id
+  FROM ${PgDatasetTableName} 
+  GROUP BY user_id,collection_id, dataset_id
+  ORDER BY dataset_id`);
+
+  if (rowCount === 0) {
+    console.log('pg done');
+    return;
+  }
+  // init dataset collection
+  console.log(`total collection: ${rowCount}`);
+
+  // collectionId 的类型：manual, mark, httpLink, fileId
+  async function initCollection(row: RowType): Promise<any> {
+    try {
+      {
+        const userId = row.user_id;
+        const datasetId = row.dataset_id;
+        const collectionId = row.collection_id;
+
+        const count = await MongoDatasetCollection.countDocuments({
+          datasetId,
+          userId,
+          ['metadata.pgCollectionId']: collectionId
+        });
+        if (count > 0) {
+          console.log('collection already exist');
+          return;
+        }
+
+        if (collectionId === 'manual') {
+          await MongoDatasetCollection.create({
+            parentId: null,
+            datasetId,
+            userId,
+            name: '手动录入',
+            type: DatasetCollectionTypeEnum.virtual,
+            updateTime: new Date('2099'),
+            metadata: {
+              pgCollectionId: collectionId
+            }
+          });
+        } else if (collectionId === 'mark') {
+          await MongoDatasetCollection.create({
+            parentId: null,
+            datasetId,
+            userId,
+            name: '手动标注',
+            type: DatasetCollectionTypeEnum.virtual,
+            updateTime: new Date('2099'),
+            metadata: {
+              pgCollectionId: collectionId
+            }
+          });
+        } else if (strIsLink(collectionId)) {
+          await MongoDatasetCollection.create({
+            parentId: null,
+            datasetId,
+            userId,
+            name: collectionId,
+            type: DatasetCollectionTypeEnum.link,
+            metadata: {
+              rawLink: collectionId,
+              pgCollectionId: collectionId
+            }
+          });
+        } else {
+          // find file
+          const gridFs = new GridFSStorage('dataset', userId);
+          const collection = gridFs.Collection();
+          const file = await collection.findOne({
+            _id: new Types.ObjectId(collectionId)
+          });
+
+          if (file) {
+            await MongoDatasetCollection.create({
+              parentId: null,
+              datasetId,
+              userId,
+              name: file.filename,
+              type: DatasetCollectionTypeEnum.file,
+              metadata: {
+                fileId: file._id,
+                pgCollectionId: collectionId
+              }
+            });
+          } else {
+            // no file
+            await MongoDatasetCollection.create({
+              parentId: null,
+              datasetId,
+              userId,
+              name: '未知文件',
+              type: DatasetCollectionTypeEnum.virtual,
+              metadata: {
+                pgCollectionId: collectionId
+              }
+            });
+          }
+        }
+        console.log('create collection success');
+      }
+    } catch (error) {
+      console.log(error);
+
+      await delay(2000);
+      return initCollection(row);
+    }
+  }
+
+  for await (const row of rows) {
+    await initCollection(row);
+    console.log('init collection success: ', ++success);
+  }
+}
+
+async function updatePgCollection(): Promise<any> {
+  let success = 0;
+  const limit = 10;
+  const collections = await MongoDatasetCollection.find({
+    'metadata.pgCollectionId': { $exists: true, $ne: '' }
+  }).lean();
+  console.log('total:', collections.length);
+
+  async function update(i: number): Promise<any> {
+    const item = collections[i];
+    if (!item) return;
+
+    try {
+      console.log('start', item.name, item.datasetId, item.metadata.pgCollectionId);
+      const time = Date.now();
+      if (item.metadata.pgCollectionId) {
+        const { rows } = await PgClient.select(PgDatasetTableName, {
+          fields: ['id'],
+          where: [
+            ['dataset_id', String(item.datasetId)],
+            'AND',
+            ['collection_id', String(item.metadata.pgCollectionId)]
+          ],
+          limit: 999999
+        });
+        console.log('update date total', rows.length, 'time:', Date.now() - time);
+
+        await PgClient.query(`
+    update ${PgDatasetTableName} set collection_id = '${item._id}' where dataset_id = '${String(
+      item.datasetId
+    )}' AND collection_id = '${String(item.metadata.pgCollectionId)}'
+              `);
+
+        console.log('pg update time', Date.now() - time);
+      }
+
+      // 更新 file id
+      if (item.type === 'file' && item.metadata.fileId) {
+        const collection = connectionMongo.connection.db.collection(`dataset.files`);
+        await collection.findOneAndUpdate({ _id: new Types.ObjectId(item.metadata.fileId) }, [
+          {
+            $set: {
+              'metadata.datasetId': item.datasetId,
+              'metadata.collectionId': item._id
+            }
+          }
+        ]);
+      }
+
+      await MongoDatasetCollection.findByIdAndUpdate(item._id, {
+        $unset: { 'metadata.pgCollectionId': '' }
+      });
+      console.log('success', ++success);
+
+      return update(i + limit);
+    } catch (error) {
+      console.log(error);
+
+      await delay(5000);
+      return update(i);
+    }
+  }
+
+  const arr = new Array(limit).fill(0);
+
+  return Promise.all(arr.map((_, i) => update(i)));
+}

projects/app/src/pages/api/admin/sqlTest.ts

@@ -1,45 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
-import { connectToDatabase } from '@/service/mongo';
-import { PgClient } from '@/service/pg';
-import { PgDatasetTableName } from '@/constants/plugin';
-import { DatasetSpecialIdEnum } from '@fastgpt/core/dataset/constant';
-import { Types, connectionMongo } from '@fastgpt/common/mongo';
-import { delay } from '@/utils/tools';
-import { replaceVariable } from '@/utils/common/tools/text';
-import { getVector } from '../openapi/plugin/vector';
-
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    // await connectToDatabase();
-    // const { text, analyze, sql } = req.body as {
-    //   userId: string;
-    //   text: string;
-    //   analyze?: boolean;
-    //   sql: string;
-    // };
-    // await authUser({ req, authRoot: true });
-
-    // const vectorModel = global.vectorModels[0];
-    // const { vectors } = await getVector({
-    //   model: vectorModel.model,
-    //   input: [text]
-    // });
-
-    // const start = Date.now();
-    // const result: any = await PgClient.query(sql.replace(/\[vector\]/g, `[${vectors[0]}]`));
-
-    jsonRes(res, {
-      data: {
-        // rows: result?.[2]?.rows,
-        // time: Date.now() - start
-      }
-    });
-  } catch (error) {
-    jsonRes(res, {
-      code: 500,
-      error
-    });
-  }
-}

projects/app/src/pages/api/app/create.ts

@@ -2,7 +2,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { App } from '@/service/models/app';
 import type { CreateAppParams } from '@/types/app';
 import { AppTypeEnum } from '@/constants/app';

projects/app/src/pages/api/app/data/totalUsage.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Bill } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
-import { Types } from '@fastgpt/common/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { Types } from '@fastgpt/service/common/mongo';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/app/del.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Chat, App, connectToDatabase, Collection } from '@/service/mongo';
-import { MongoOutLink } from '@fastgpt/support/outLink/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoOutLink } from '@fastgpt/service/support/outLink/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { authApp } from '@/service/utils/auth';
 
 /* 获取我的模型 */

projects/app/src/pages/api/app/detail.tsx

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { authApp } from '@/service/utils/auth';
 
 /* 获取我的模型 */

projects/app/src/pages/api/app/getChatLogs.ts

@@ -1,10 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Chat, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { PagingData } from '@/types';
 import { AppLogsListItemType } from '@/types/app';
-import { Types } from '@fastgpt/common/mongo';
+import { Types } from '@fastgpt/service/common/mongo';
 import { addDays } from 'date-fns';
 import type { GetAppChatLogsParams } from '@/global/core/api/appReq.d';
 

projects/app/src/pages/api/app/myApps.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, App } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { AppListItemType } from '@/types/app';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/app/share/collection.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Collection, App } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 /* 模型收藏切换 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/app/share/getModels.ts

@@ -3,8 +3,8 @@ import { jsonRes } from '@/service/response';
 import { connectToDatabase, App } from '@/service/mongo';
 import type { PagingData } from '@/types';
 import type { ShareAppItem } from '@/types/app';
-import { authUser } from '@fastgpt/support/user/auth';
-import { Types } from '@fastgpt/common/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { Types } from '@fastgpt/service/common/mongo';
 
 /* 获取模型列表 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {

projects/app/src/pages/api/app/update.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { App } from '@/service/models/app';
 import type { AppUpdateParams } from '@/types/app';
 import { authApp } from '@/service/utils/auth';

projects/app/src/pages/api/chat/chatTest.ts

@@ -1,9 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { sseErrRes } from '@/service/response';
 import { sseResponseEventEnum } from '@/constants/chat';
-import { responseWrite } from '@fastgpt/common/tools/stream';
+import { responseWrite } from '@fastgpt/service/common/response';
 import { AppModuleItemType } from '@/types/app';
 import { dispatchModules } from '@/pages/api/v1/chat/completions';
 import { pushChatBill } from '@/service/common/bill/push';

projects/app/src/pages/api/chat/delChatRecordByContentId.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, ChatItem } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/chat/feedback/adminUpdate.ts

@@ -2,15 +2,15 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, ChatItem } from '@/service/mongo';
 import type { AdminUpdateFeedbackParams } from '@/global/core/api/chatReq.d';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 /* 初始化我的聊天框，需要身份验证 */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {
     await connectToDatabase();
-    const { chatItemId, kbId, dataId, content = undefined } = req.body as AdminUpdateFeedbackParams;
+    const { chatItemId, datasetId, dataId, q, a } = req.body as AdminUpdateFeedbackParams;
 
-    if (!chatItemId || !kbId || !dataId || !content) {
+    if (!chatItemId || !datasetId || !dataId || !q) {
       throw new Error('missing parameter');
     }
 
@@ -23,9 +23,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       },
       {
         adminFeedback: {
-          kbId,
+          datasetId,
           dataId,
-          content
+          q,
+          a
         }
       }
     );

projects/app/src/pages/api/chat/history/getHistory.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { ChatHistoryItemType } from '@/types/chat';
 import { ChatSourceEnum } from '@/constants/chat';
 

projects/app/src/pages/api/chat/history/updateChatHistory.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 export type Props = {
   chatId: string;

projects/app/src/pages/api/chat/init.ts

@@ -2,11 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { Chat, ChatItem, connectToDatabase } from '@/service/mongo';
 import type { InitChatResponse } from '@/global/core/api/chatRes.d';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { ChatItemType } from '@/types/chat';
 import { authApp } from '@/service/utils/auth';
 import type { ChatSchema } from '@/types/mongoSchema';
-import { getGuideModule } from '@/components/ChatBox/utils';
+import { getGuideModule } from '@/global/core/app/modules/utils';
 import { getChatModelNameListByModules } from '@/service/core/app/module';
 import { TaskResponseKeyEnum } from '@/constants/chat';
 

projects/app/src/pages/api/chat/removeHistory.ts

@@ -1,7 +1,7 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Chat, ChatItem } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { ChatSourceEnum } from '@/constants/chat';
 
 type Props = {

projects/app/src/pages/api/common/bill/createTrainingBill.ts

@@ -1,9 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase, Bill } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { BillSourceEnum } from '@/constants/user';
-import { CreateTrainingBillType } from '@/global/common/api/billReq.d';
+import { CreateTrainingBillType } from '@fastgpt/global/common/bill/types/billReq.d';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
   try {

projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts

@@ -1,10 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { CreateQuestionGuideParams } from '@/global/core/api/aiReq.d';
 import { pushQuestionGuideBill } from '@/service/common/bill/push';
-import { createQuestionGuide } from '@fastgpt/core/ai/functions/createQuestionGuide';
+import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {

projects/app/src/pages/api/core/dataset/allDataset.ts

@@ -1,8 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { getVectorModel } from '@/service/core/ai/model';
 import type { DatasetsItemType } from '@/types/core/dataset';
 
@@ -12,12 +12,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    const kbList = await MongoDataset.find({
+    const datasets = await MongoDataset.find({
       userId,
       type: 'dataset'
     });
 
-    const data = kbList.map((item) => ({
+    const data = datasets.map((item) => ({
       ...item.toJSON(),
       vectorModel: getVectorModel(item.vectorModel)
     }));

projects/app/src/pages/api/core/dataset/collection/create.ts

@@ -0,0 +1,87 @@
+/* 
+    Create one dataset collection
+*/
+
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import type { CreateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    const { userId } = await authUser({ req, authToken: true });
+
+    const body = req.body || {};
+
+    jsonRes(res, {
+      data: await createOneCollection({
+        ...body,
+        userId
+      })
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}
+
+export async function createOneCollection({
+  name,
+  parentId,
+  datasetId,
+  type,
+  metadata = {},
+  userId
+}: CreateDatasetCollectionParams & { userId: string }) {
+  const { _id } = await MongoDatasetCollection.create({
+    name,
+    userId,
+    datasetId,
+    parentId: parentId || null,
+    type,
+    metadata,
+    updateTime: getCollectionUpdateTime({ name })
+  });
+
+  // create default collection
+  if (type === DatasetCollectionTypeEnum.folder) {
+    await createDefaultCollection({
+      datasetId,
+      parentId: _id,
+      userId
+    });
+  }
+
+  return _id;
+}
+
+// create default collection
+export function createDefaultCollection({
+  name = '手动录入',
+  datasetId,
+  parentId,
+  userId
+}: {
+  name?: '手动录入' | '手动标注';
+  datasetId: string;
+  parentId?: string;
+  userId: string;
+}) {
+  return MongoDatasetCollection.create({
+    name,
+    userId,
+    datasetId,
+    parentId,
+    type: DatasetCollectionTypeEnum.virtual,
+    updateTime: new Date('2000'),
+    metadata: {}
+  });
+}

projects/app/src/pages/api/core/dataset/collection/delById.ts

@@ -0,0 +1,61 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils';
+import { delDataByCollectionId } from '@/service/core/dataset/data/utils';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { GridFSStorage } from '@/service/lib/gridfs';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    const { collectionId } = req.query as { collectionId: string };
+
+    if (!collectionId) {
+      throw new Error('CollectionIdId is required');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    // find all delete id
+    const collections = await findCollectionAndChild(collectionId, '_id metadata');
+    const delIdList = collections.map((item) => item._id);
+
+    // delete pg data
+    await delDataByCollectionId({ userId, collectionIds: delIdList });
+
+    // delete training data
+    await MongoDatasetTraining.deleteMany({
+      datasetCollectionId: { $in: delIdList },
+      userId
+    });
+
+    // delete file
+    const gridFs = new GridFSStorage('dataset', userId);
+    const fileCollection = gridFs.Collection();
+    await Promise.all(
+      collections.map(
+        (item) =>
+          //@ts-ignore
+          item.metadata?.fileId && fileCollection.findOneAndDelete({ _id: item.metadata.fileId })
+      )
+    );
+
+    // delete collection
+    await MongoDatasetCollection.deleteMany({
+      _id: { $in: delIdList },
+      userId
+    });
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/collection/detail.ts

@@ -0,0 +1,37 @@
+/* 
+    Get one dataset collection detail
+*/
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+    const { id } = req.query as { id: string };
+
+    if (!id) {
+      throw new Error('Id is required');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    const collection = await MongoDatasetCollection.findOne({ _id: id, userId }).lean();
+
+    if (!collection) {
+      throw new Error('Collection not found');
+    }
+
+    jsonRes(res, {
+      data: collection
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/collection/list.ts

@@ -0,0 +1,134 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { DatasetTrainingCollectionName } from '@fastgpt/service/core/dataset/training/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
+
+import { Types } from '@fastgpt/service/common/mongo';
+import type { DatasetCollectionsListItemType } from '@/global/core/dataset/response';
+import type { GetDatasetCollectionsProps } from '@/global/core/api/datasetReq';
+import { PagingData } from '@/types';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { countCollectionData } from '@/service/core/dataset/data/utils';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { startQueue } from '@/service/utils/tools';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    let {
+      pageNum = 1,
+      pageSize = 10,
+      datasetId,
+      parentId = null,
+      searchText = '',
+      selectFolder = false,
+      simple = false
+    } = req.body as GetDatasetCollectionsProps;
+    searchText = searchText?.replace(/'/g, '');
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    const match = {
+      userId: new Types.ObjectId(userId),
+      datasetId: new Types.ObjectId(datasetId),
+      parentId: parentId ? new Types.ObjectId(parentId) : null,
+      ...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}),
+      ...(searchText
+        ? {
+            name: new RegExp(searchText, 'i')
+          }
+        : {})
+    };
+
+    if (simple) {
+      const collections = await MongoDatasetCollection.find(match, '_id name type parentId')
+        .sort({
+          updateTime: -1
+        })
+        .lean();
+      return jsonRes<PagingData<DatasetCollectionsListItemType>>(res, {
+        data: {
+          pageNum,
+          pageSize,
+          data: await Promise.all(
+            collections.map(async (item) => ({
+              ...item,
+              dataAmount: 0,
+              trainingAmount: 0
+            }))
+          ),
+          total: await MongoDatasetCollection.countDocuments(match)
+        }
+      });
+    }
+
+    const collections = await MongoDatasetCollection.aggregate([
+      {
+        $match: match
+      },
+      {
+        $lookup: {
+          from: DatasetTrainingCollectionName,
+          localField: '_id',
+          foreignField: 'datasetCollectionId',
+          as: 'trainings_amount'
+        }
+      },
+      // 统计子集合的数量和子训练的数量
+      {
+        $project: {
+          _id: 1,
+          parentId: 1,
+          fileId: 1,
+          name: 1,
+          type: 1,
+          updateTime: 1,
+          trainingAmount: { $size: '$trainings_amount' }
+        }
+      },
+      {
+        $sort: { updateTime: -1 }
+      },
+      {
+        $skip: (pageNum - 1) * pageSize
+      },
+      {
+        $limit: pageSize
+      }
+    ]);
+
+    const counts = await countCollectionData({
+      collectionIds: collections.map((item) => item._id),
+      datasetId
+    });
+
+    const data = await Promise.all(
+      collections.map(async (item, i) => ({
+        ...item,
+        dataAmount: item.type === DatasetCollectionTypeEnum.folder ? undefined : counts[i]
+      }))
+    );
+
+    if (data.find((item) => item.trainingAmount > 0)) {
+      startQueue(1);
+    }
+
+    // count collections
+    jsonRes<PagingData<DatasetCollectionsListItemType>>(res, {
+      data: {
+        pageNum,
+        pageSize,
+        data,
+        total: await MongoDatasetCollection.countDocuments(match)
+      }
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/collection/paths.ts

@@ -0,0 +1,29 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import type { DatasetPathItemType } from '@/types/core/dataset';
+import { getDatasetCollectionPaths } from '@fastgpt/service/core/dataset/collection/utils';
+import { authUser } from '@fastgpt/service/support/user/auth';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+
+    const { parentId } = req.query as { parentId: string };
+
+    const { userId } = await authUser({ req, authToken: true });
+    const paths = await getDatasetCollectionPaths({
+      parentId,
+      userId
+    });
+
+    jsonRes<DatasetPathItemType[]>(res, {
+      data: paths
+    });
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/collection/update.ts

@@ -0,0 +1,48 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { jsonRes } from '@/service/response';
+import { connectToDatabase } from '@/service/mongo';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import type { UpdateDatasetCollectionParams } from '@/global/core/api/datasetReq.d';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { getCollectionUpdateTime } from '@fastgpt/service/core/dataset/collection/utils';
+
+export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
+  try {
+    await connectToDatabase();
+    const { id, parentId, name, metadata = {} } = req.body as UpdateDatasetCollectionParams;
+
+    if (!id) {
+      throw new Error('缺少参数');
+    }
+
+    // 凭证校验
+    const { userId } = await authUser({ req, authToken: true });
+
+    const updateFields: Record<string, any> = {
+      ...(parentId !== undefined && { parentId: parentId || null }),
+      ...(name && { name, updateTime: getCollectionUpdateTime({ name }) })
+    };
+
+    // 将metadata的每个字段添加到updateFields中
+    for (const [key, value] of Object.entries(metadata)) {
+      updateFields[`metadata.${key}`] = value;
+    }
+
+    await MongoDatasetCollection.findOneAndUpdate(
+      {
+        _id: id,
+        userId
+      },
+      {
+        $set: updateFields
+      }
+    );
+
+    jsonRes(res);
+  } catch (err) {
+    jsonRes(res, {
+      code: 500,
+      error: err
+    });
+  }
+}

projects/app/src/pages/api/core/dataset/create.ts

@@ -1,9 +1,10 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import type { CreateDatasetParams } from '@/global/core/api/datasetReq.d';
+import { createDefaultCollection } from './collection/create';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -23,6 +24,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       type
     });
 
+    await createDefaultCollection({
+      datasetId: _id,
+      userId
+    });
+
     jsonRes(res, { data: _id });
   } catch (err) {
     jsonRes(res, {

projects/app/src/pages/api/core/dataset/data/delDataById.ts

@@ -1,20 +1,20 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgClient } from '@/service/pg';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { connectToDatabase } from '@/service/mongo';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    let { dataId } = req.query as {
+    const { dataId } = req.query as {
       dataId: string;
     };
 
     if (!dataId) {
-      throw new Error('缺少参数');
+      throw new Error('dataId is required');
     }
 
     // 凭证校验

projects/app/src/pages/api/core/dataset/data/exportAll.ts

@@ -1,30 +1,30 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoUser } from '@fastgpt/support/user/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { MongoUser } from '@fastgpt/service/support/user/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { findAllChildrenIds } from '../delete';
 import QueryStream from 'pg-query-stream';
 import { PgClient } from '@/service/pg';
 import { addLog } from '@/service/utils/tools';
-import { responseWriteController } from '@fastgpt/common/tools/stream';
+import { responseWriteController } from '@fastgpt/service/common/response';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    let { kbId } = req.query as {
-      kbId: string;
+    let { datasetId } = req.query as {
+      datasetId: string;
     };
 
-    if (!kbId || !global.pgClient) {
+    if (!datasetId || !global.pgClient) {
       throw new Error('缺少参数');
     }
 
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    const exportIds = [kbId, ...(await findAllChildrenIds(kbId))];
+    const exportIds = [datasetId, ...(await findAllChildrenIds(datasetId))];
 
     const limitMinutesAgo = new Date(
       Date.now() - (global.feConfigs?.limit?.exportLimitMinutes || 0) * 60 * 1000
@@ -48,7 +48,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     }
 
     const { rows } = await PgClient.query(
-      `SELECT count(id) FROM ${PgDatasetTableName} where user_id='${userId}' AND kb_id IN (${exportIds
+      `SELECT count(id) FROM ${PgDatasetTableName} where user_id='${userId}' AND dataset_id IN (${exportIds
         .map((id) => `'${id}'`)
         .join(',')})`
     );
@@ -67,10 +67,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         res.end('Error connecting to database');
         return;
       }
+      if (!client) return;
 
       // create pg select stream
       const query = new QueryStream(
-        `SELECT q, a, source FROM ${PgDatasetTableName} where user_id='${userId}' AND kb_id IN (${exportIds
+        `SELECT q, a FROM ${PgDatasetTableName} where user_id='${userId}' AND dataset_id IN (${exportIds
           .map((id) => `'${id}'`)
           .join(',')})`
       );
@@ -84,18 +85,18 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         readStream: stream
       });
 
-      write('index,content,source');
+      write('index,content');
 
       // parse data every row
-      stream.on('data', ({ q, a, source }: { q: string; a: string; source?: string }) => {
+      stream.on('data', ({ q, a }: { q: string; a: string }) => {
         if (res.closed) {
           return stream.destroy();
         }
         q = q.replace(/"/g, '""');
         a = a.replace(/"/g, '""');
-        source = source?.replace(/"/g, '""');
+        // source = source?.replace(/"/g, '""');
 
-        write(`\n"${q}","${a || ''}","${source || ''}"`);
+        write(`\n"${q}","${a || ''}"`);
       });
       // finish
       stream.on('end', async () => {

projects/app/src/pages/api/core/dataset/data/getDataById.ts

@@ -1,10 +1,11 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
-import type { PgDataItemType } from '@/types/core/dataset/data';
+import type { DatasetDataItemType, PgDataItemType } from '@fastgpt/global/core/dataset/type';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 
 export type Response = {
   id: string;
@@ -26,16 +27,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    const where: any = [['user_id', userId], 'AND', ['id', dataId]];
-
-    const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, {
-      fields: ['kb_id', 'id', 'q', 'a', 'source', 'file_id'],
-      where,
-      limit: 1
-    });
-
     jsonRes(res, {
-      data: searchRes.rows[0]
+      data: await getDatasetDataById({ userId, id: dataId })
     });
   } catch (err) {
     jsonRes(res, {
@@ -44,3 +37,70 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     });
   }
 }
+
+export async function getDatasetDataById({
+  id,
+  userId
+}: {
+  id: string;
+  userId: string;
+}): Promise<DatasetDataItemType> {
+  const where: any = [['user_id', userId], 'AND', ['id', id]];
+
+  const searchRes = await PgClient.select<PgDataItemType>(PgDatasetTableName, {
+    fields: ['id', 'q', 'a', 'dataset_id', 'collection_id'],
+    where,
+    limit: 1
+  });
+
+  const data = searchRes?.rows?.[0];
+
+  if (!data) {
+    return Promise.reject('Data not found');
+  }
+
+  // find source
+  const collection = (await getDatasetDataItemInfo({ pgDataList: [data] }))[0];
+
+  if (!collection) {
+    return Promise.reject('Data Collection not found');
+  }
+
+  return {
+    id: data.id,
+    q: data.q,
+    a: data.a,
+    datasetId: data.dataset_id,
+    collectionId: data.collection_id,
+    sourceName: collection.sourceName,
+    sourceId: collection.sourceId
+  };
+}
+
+export async function getDatasetDataItemInfo({
+  pgDataList
+}: {
+  pgDataList: PgDataItemType[];
+}): Promise<DatasetDataItemType[]> {
+  const collections = await MongoDatasetCollection.find(
+    {
+      _id: { $in: pgDataList.map((item) => item.collection_id) }
+    },
+    '_id name datasetId metadata'
+  ).lean();
+
+  return pgDataList.map((item) => {
+    const collection = collections.find(
+      (collection) => String(collection._id) === item.collection_id
+    );
+    return {
+      id: item.id,
+      q: item.q,
+      a: item.a,
+      datasetId: collection?.datasetId || '',
+      collectionId: item.collection_id,
+      sourceName: collection?.name || '',
+      sourceId: collection?.metadata?.fileId || collection?.metadata?.rawLink
+    };
+  });
+}

projects/app/src/pages/api/core/dataset/data/getDataList.ts

@@ -1,29 +1,23 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
-import type { PgDataItemType } from '@/types/core/dataset/data';
+import type { DatasetDataListItemType } from '@/global/core/dataset/response.d';
+import type { GetDatasetDataListProps } from '@/global/core/api/datasetReq';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
     let {
-      kbId,
       pageNum = 1,
       pageSize = 10,
       searchText = '',
-      fileId = ''
-    } = req.body as {
-      kbId: string;
-      pageNum: number;
-      pageSize: number;
-      searchText: string;
-      fileId: string;
-    };
-    if (!kbId) {
-      throw new Error('缺少参数');
+      collectionId
+    } = req.body as GetDatasetDataListProps;
+    if (!collectionId) {
+      throw new Error('collectionId is required');
     }
 
     // 凭证校验
@@ -34,20 +28,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     const where: any = [
       ['user_id', userId],
       'AND',
-      ['kb_id', kbId],
-      'AND',
-      ['file_id', fileId],
-      ...(searchText
-        ? [
-            'AND',
-            `(q ILIKE '%${searchText}%' OR a ILIKE '%${searchText}%' OR source ILIKE '%${searchText}%')`
-          ]
-        : [])
+      ['collection_id', collectionId],
+      searchText ? `AND (q ILIKE '%${searchText}%' OR a ILIKE '%${searchText}%')` : ''
     ];
 
     const [searchRes, total] = await Promise.all([
-      PgClient.select<PgDataItemType>(PgDatasetTableName, {
-        fields: ['id', 'q', 'a', 'source', 'file_id'],
+      PgClient.select<DatasetDataListItemType>(PgDatasetTableName, {
+        fields: ['id', 'q', 'a'],
         where,
         order: [{ field: 'id', mode: 'DESC' }],
         limit: pageSize,

projects/app/src/pages/api/core/dataset/data/getQueueLen.ts

@@ -1,7 +1,8 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { TrainingData, connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 
 /* 拆分数据成QA */
 export default async function handler(req: NextApiRequest, res: NextApiResponse) {
@@ -10,7 +11,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
     await authUser({ req, authToken: true });
 
     // split queue data
-    const result = await TrainingData.countDocuments({
+    const result = await MongoDatasetTraining.countDocuments({
       lockTime: { $lt: new Date('2040/1/1') }
     });
 

projects/app/src/pages/api/core/dataset/data/getTrainingData.ts

@@ -1,52 +0,0 @@
-import type { NextApiRequest, NextApiResponse } from 'next';
-import { jsonRes } from '@/service/response';
-import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
-import { TrainingModeEnum } from '@/constants/plugin';
-import { Types } from '@fastgpt/common/mongo';
-import { startQueue } from '@/service/utils/tools';
-
-/* 拆分数据成QA */
-export default async function handler(req: NextApiRequest, res: NextApiResponse) {
-  try {
-    await connectToDatabase();
-    const { kbId, init = false } = req.body as { kbId: string; init: boolean };
-    if (!kbId) {
-      throw new Error('参数错误');
-    }
-
-    const { userId } = await authUser({ req, authToken: true });
-
-    // split queue data
-    const result = await TrainingData.aggregate([
-      {
-        $match: {
-          userId: new Types.ObjectId(userId),
-          kbId: new Types.ObjectId(kbId)
-        }
-      },
-      {
-        $group: {
-          _id: '$mode',
-          count: { $sum: 1 }
-        }
-      }
-    ]);
-
-    jsonRes(res, {
-      data: {
-        qaListLen: result.find((item) => item._id === TrainingModeEnum.qa)?.count || 0,
-        vectorListLen: result.find((item) => item._id === TrainingModeEnum.index)?.count || 0
-      }
-    });
-
-    if (init) {
-      startQueue();
-    }
-  } catch (err) {
-    jsonRes(res, {
-      code: 500,
-      error: err
-    });
-  }
-}

projects/app/src/pages/api/core/dataset/data/insertData.ts

@@ -5,21 +5,15 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authDataset } from '@/service/utils/auth';
-import { authUser } from '@fastgpt/support/user/auth';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
-import { PgDatasetTableName } from '@/constants/plugin';
-import { insertData2Dataset, PgClient } from '@/service/pg';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
+import { SetOneDatasetDataProps } from '@/global/core/api/datasetReq';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
+import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant';
+import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type';
+import { countPromptTokens } from '@/global/common/tiktoken';
 import { getVectorModel } from '@/service/core/ai/model';
-import { getVector } from '@/pages/api/openapi/plugin/vector';
-import { DatasetDataItemType } from '@/types/core/dataset/data';
-import { countPromptTokens } from '@/utils/common/tiktoken';
-import { authFileIdValid } from '@/service/dataset/auth';
-
-export type Props = {
-  kbId: string;
-  data: DatasetDataItemType;
-};
+import { insertData2Dataset, hasSameValue } from '@/service/core/dataset/data/utils';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -28,7 +22,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    jsonRes(res, {
+    jsonRes<string>(res, {
       data: await getVectorAndInsertDataset({
         ...req.body,
         userId
@@ -43,58 +37,59 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
 });
 
 export async function getVectorAndInsertDataset(
-  props: Props & { userId: string }
+  props: SetOneDatasetDataProps & { userId: string }
 ): Promise<string> {
-  const { kbId, data, userId } = props;
-  if (!kbId || !data?.q) {
-    return Promise.reject('缺少参数');
+  let { datasetId, collectionId, q, a, userId } = props;
+
+  if (!datasetId) {
+    return Promise.reject('知识库 ID 不能为空');
   }
 
-  // auth kb
-  const kb = await authDataset({ kbId, userId });
+  if (!q) {
+    return Promise.reject('索引内容不能为空');
+  }
 
-  const q = data?.q?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
-  const a = data?.a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
+  if (!collectionId) {
+    return Promise.reject('集合 ID 和集合类型不能同时为空');
+  }
+
+  // auth collection and get dataset
+  const collection = await MongoDatasetCollection.findOne({
+    _id: collectionId,
+    userId,
+    datasetId,
+    type: { $ne: DatasetCollectionTypeEnum.folder }
+  }).populate('datasetId', '_id vectorModel');
+
+  if (!collection) {
+    return Promise.reject('集合不存在');
+  }
+  const dataset = collection.datasetId as unknown as DatasetSchemaType;
+
+  // format data
+  const formatQ = q?.replace(/\\n/g, '\n').trim().replace(/'/g, '"');
+  const formatA = a?.replace(/\\n/g, '\n').trim().replace(/'/g, '"') || '';
 
   // token check
-  const token = countPromptTokens(q, 'system');
+  const token = countPromptTokens(formatQ, 'system');
 
-  if (token > getVectorModel(kb.vectorModel).maxToken) {
-    return Promise.reject('Over Tokens');
+  if (token > getVectorModel(dataset.vectorModel).maxToken) {
+    return Promise.reject('Q Over Tokens');
   }
 
-  const { rows: existsRows } = await PgClient.query(`
-  SELECT COUNT(*) > 0 AS exists
-  FROM  ${PgDatasetTableName} 
-  WHERE md5(q)=md5('${q}') AND md5(a)=md5('${a}') AND user_id='${userId}' AND file_id='${data.file_id}' AND kb_id='${kbId}'
-`);
-  const exists = existsRows[0]?.exists || false;
-
-  if (exists) {
-    return Promise.reject('已经存在完全一致的数据');
-  }
-
-  await authFileIdValid(data.file_id);
-
-  const { vectors } = await getVector({
-    model: kb.vectorModel,
-    input: [q],
-    userId
+  // Duplicate data check
+  await hasSameValue({
+    collectionId,
+    q,
+    a
   });
 
-  const response = await insertData2Dataset({
+  return insertData2Dataset({
     userId,
-    kbId,
-    data: [
-      {
-        ...data,
-        q,
-        a,
-        vector: vectors[0]
-      }
-    ]
+    q: formatQ,
+    a: formatA,
+    collectionId,
+    datasetId,
+    model: dataset.vectorModel
   });
-
-  // @ts-ignore
-  return response?.rows?.[0]?.id || '';
 }

projects/app/src/pages/api/core/dataset/data/pushData.ts

@@ -1,18 +1,17 @@
 /* push data to training queue */
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authUser } from '@fastgpt/support/user/auth';
-import { authDataset } from '@/service/utils/auth';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
-import { TrainingModeEnum } from '@/constants/plugin';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { authCollection } from '@fastgpt/service/core/dataset/auth';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
+import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant';
 import { startQueue } from '@/service/utils/tools';
-import { DatasetDataItemType } from '@/types/core/dataset/data';
-import { countPromptTokens } from '@/utils/common/tiktoken';
+import { DatasetChunkItemType } from '@fastgpt/global/core/dataset/type';
+import { countPromptTokens } from '@/global/common/tiktoken';
 import type { PushDataResponse } from '@/global/core/api/datasetRes.d';
 import type { PushDataProps } from '@/global/core/api/datasetReq.d';
-import { authFileIdValid } from '@/service/dataset/auth';
 import { getVectorModel } from '@/service/core/ai/model';
 
 const modeMap = {
@@ -23,25 +22,25 @@ const modeMap = {
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    const { kbId, data, mode = TrainingModeEnum.index } = req.body as PushDataProps;
+    const { collectionId, data, mode = TrainingModeEnum.index } = req.body as PushDataProps;
 
-    if (!kbId || !Array.isArray(data)) {
-      throw new Error('KbId or data is empty');
+    if (!collectionId || !Array.isArray(data)) {
+      throw new Error('collectionId or data is empty');
     }
 
     if (modeMap[mode] === undefined) {
-      throw new Error('Mode is error');
+      throw new Error('Mode is not index or qa');
     }
 
-    if (data.length > 500) {
-      throw new Error('Data is too long, max 500');
+    if (data.length > 200) {
+      throw new Error('Data is too long, max 200');
     }
 
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true, authApiKey: true });
 
     jsonRes<PushDataResponse>(res, {
-      data: await pushDataToKb({
+      data: await pushDataToDatasetCollection({
         ...req.body,
         userId
       })
@@ -54,40 +53,40 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex
   }
 });
 
-export async function pushDataToKb({
+export async function pushDataToDatasetCollection({
   userId,
-  kbId,
+  collectionId,
   data,
   mode,
   prompt,
   billId
 }: { userId: string } & PushDataProps): Promise<PushDataResponse> {
-  const [kb, vectorModel] = await Promise.all([
-    authDataset({
-      userId,
-      kbId
-    }),
-    (async () => {
-      if (mode === TrainingModeEnum.index) {
-        const vectorModel = (await MongoDataset.findById(kbId, 'vectorModel'))?.vectorModel;
+  // auth dataset & get training model
+  const {
+    dataset: { _id: datasetId, vectorModel }
+  } = await authCollection({
+    userId,
+    collectionId
+  });
+  const vectorModelData = getVectorModel(vectorModel);
 
-        return getVectorModel(vectorModel);
-      }
-      return global.vectorModels[0];
-    })()
-  ]);
-
-  const modeMaxToken = {
-    [TrainingModeEnum.index]: vectorModel.maxToken * 1.5,
-    [TrainingModeEnum.qa]: global.qaModels[0].maxToken * 0.8
+  const modeMap = {
+    [TrainingModeEnum.index]: {
+      maxToken: vectorModelData.maxToken * 1.5,
+      model: vectorModelData.model
+    },
+    [TrainingModeEnum.qa]: {
+      maxToken: global.qaModels[0].maxToken * 0.8,
+      model: global.qaModels[0].model
+    }
   };
 
   // filter repeat or equal content
   const set = new Set();
-  const filterResult: Record<string, DatasetDataItemType[]> = {
+  const filterResult: Record<string, DatasetChunkItemType[]> = {
     success: [],
     overToken: [],
-    fileIdInvalid: [],
+    repeat: [],
     error: []
   };
 
@@ -101,21 +100,16 @@ export async function pushDataToKb({
       const text = item.q + item.a;
 
       // count q token
-      const token = countPromptTokens(item.q, 'system');
+      const token = countPromptTokens(item.q);
 
-      if (token > modeMaxToken[mode]) {
+      if (token > modeMap[mode].maxToken) {
         filterResult.overToken.push(item);
         return;
       }
 
-      try {
-        await authFileIdValid(item.file_id);
-      } catch (error) {
-        filterResult.fileIdInvalid.push(item);
-        return;
-      }
-
-      if (!set.has(text)) {
+      if (set.has(text)) {
+        filterResult.repeat.push(item);
+      } else {
         filterResult.success.push(item);
         set.add(text);
       }
@@ -123,15 +117,17 @@ export async function pushDataToKb({
   );
 
   // 插入记录
-  const insertRes = await TrainingData.insertMany(
+  const insertRes = await MongoDatasetTraining.insertMany(
     filterResult.success.map((item) => ({
-      ...item,
       userId,
-      kbId,
+      datasetId,
+      datasetCollectionId: collectionId,
+      billId,
       mode,
       prompt,
-      billId,
-      vectorModel: vectorModel.model
+      model: modeMap[mode].model,
+      q: item.q,
+      a: item.a
     }))
   );
 

projects/app/src/pages/api/core/dataset/data/updateData.ts

@@ -1,57 +1,37 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { authUser } from '@fastgpt/support/user/auth';
-import { PgClient } from '@/service/pg';
-import { withNextCors } from '@fastgpt/common/tools/nextjs';
+import { authUser } from '@fastgpt/service/support/user/auth';
+import { withNextCors } from '@fastgpt/service/common/middle/cors';
 import { connectToDatabase } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { getVector } from '@/pages/api/openapi/plugin/vector';
-import { PgDatasetTableName } from '@/constants/plugin';
-import type { UpdateDatasetDataPrams } from '@/global/core/api/datasetReq.d';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import type { SetOneDatasetDataProps } from '@/global/core/api/datasetReq.d';
+import { updateData2Dataset } from '@/service/core/dataset/data/utils';
 
 export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
     await connectToDatabase();
-    const { dataId, a = '', q = '', kbId } = req.body as UpdateDatasetDataPrams;
+    const { id, datasetId, collectionId, q = '', a } = req.body as SetOneDatasetDataProps;
 
-    if (!dataId) {
+    if (!id || !collectionId) {
       throw new Error('缺少参数');
     }
 
     // auth user and get kb
-    const [{ userId }, kb] = await Promise.all([
+    const [{ userId }, dataset] = await Promise.all([
       authUser({ req, authToken: true }),
-      MongoDataset.findById(kbId, 'vectorModel')
+      MongoDataset.findById(datasetId, 'vectorModel')
     ]);
 
-    if (!kb) {
+    if (!dataset) {
       throw new Error("Can't find database");
     }
 
-    // get vector
-    const { vectors = [] } = await (async () => {
-      if (q) {
-        return getVector({
-          userId,
-          input: [q],
-          model: kb.vectorModel
-        });
-      }
-      return { vectors: [[]] };
-    })();
-
-    // 更新 pg 内容.仅修改a，不需要更新向量。
-    await PgClient.update(PgDatasetTableName, {
-      where: [['id', dataId], 'AND', ['user_id', userId]],
-      values: [
-        { key: 'a', value: a.replace(/'/g, '"') },
-        ...(q
-          ? [
-              { key: 'q', value: q.replace(/'/g, '"') },
-              { key: 'vector', value: `[${vectors[0]}]` }
-            ]
-          : [])
-      ]
+    await updateData2Dataset({
+      dataId: id,
+      userId,
+      q,
+      a,
+      model: dataset.vectorModel
     });
 
     jsonRes(res);

projects/app/src/pages/api/core/dataset/delete.ts

@@ -1,12 +1,14 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
-import { connectToDatabase, TrainingData } from '@/service/mongo';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
-import { authUser } from '@fastgpt/support/user/auth';
+import { connectToDatabase } from '@/service/mongo';
+import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { PgClient } from '@/service/pg';
 import { PgDatasetTableName } from '@/constants/plugin';
 import { GridFSStorage } from '@/service/lib/gridfs';
-import { Types } from '@fastgpt/common/mongo';
+import { Types } from '@fastgpt/service/common/mongo';
+import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
@@ -25,9 +27,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     const deletedIds = [id, ...(await findAllChildrenIds(id))];
 
     // delete training data
-    await TrainingData.deleteMany({
+    await MongoDatasetTraining.deleteMany({
       userId,
-      kbId: { $in: deletedIds.map((id) => new Types.ObjectId(id)) }
+      datasetId: { $in: deletedIds.map((id) => new Types.ObjectId(id)) }
     });
 
     // delete all pg data
@@ -35,15 +37,20 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
       where: [
         ['user_id', userId],
         'AND',
-        `kb_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`
+        `dataset_id IN (${deletedIds.map((id) => `'${id}'`).join(',')})`
       ]
     });
 
     // delete related files
     const gridFs = new GridFSStorage('dataset', userId);
-    await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByKbId(id)));
+    await Promise.all(deletedIds.map((id) => gridFs.deleteFilesByDatasetId(id)));
 
-    // delete kb data
+    // delete collections
+    await MongoDatasetCollection.deleteMany({
+      datasetId: { $in: deletedIds }
+    });
+
+    // delete dataset data
     await MongoDataset.deleteMany({
       _id: { $in: deletedIds },
       userId

projects/app/src/pages/api/core/dataset/detail.ts

@@ -1,9 +1,9 @@
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { connectToDatabase } from '@/service/mongo';
-import { authUser } from '@fastgpt/support/user/auth';
+import { authUser } from '@fastgpt/service/support/user/auth';
 import { getVectorModel } from '@/service/core/ai/model';
-import { MongoDataset } from '@fastgpt/core/dataset/schema';
+import { MongoDataset } from '@fastgpt/service/core/dataset/schema';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {