export data limit file_id

client/data/config.json

@@ -9,7 +9,7 @@
     "show_doc": true,
     "systemTitle": "FastGPT",
     "authorText": "Made by FastGPT Team.",
-    "gitLoginKey": "",
+    "exportLimitMinutes": 0,
     "scripts": []
   },
   "SystemParams": {
@@ -61,4 +61,4 @@
     "maxToken": 16000,
     "price": 0
   }
-}
+}

client/src/api/plugins/kb.ts

@@ -53,14 +53,10 @@ export const getKbDataList = (data: GetKbDataListProps) =>
 /**
  * 获取导出数据（不分页）
  */
-export const getExportDataList = (kbId: string) =>
+export const getExportDataList = (data: { kbId: string; fileId: string }) =>
-  GET<[string, string, string][]>(
+  GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, {
-    `/plugins/kb/data/exportModelData`,
+    timeout: 600000
-    { kbId },
+  });
-    {
-      timeout: 600000
-    }
-  );
 
 /**
  * 获取模型正在拆分数据的数量

client/src/pages/api/plugins/kb/data/exportModelData.ts

@@ -4,11 +4,13 @@ import { connectToDatabase, User } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { PgClient } from '@/service/pg';
 import { PgTrainingTableName } from '@/constants/plugin';
+import { OtherFileId } from '@/constants/kb';
 
 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
   try {
-    let { kbId } = req.query as {
+    let { kbId, fileId } = req.query as {
       kbId: string;
+      fileId: string;
     };
 
     if (!kbId) {
@@ -20,7 +22,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     // 凭证校验
     const { userId } = await authUser({ req, authToken: true });
 
-    const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000);
+    const thirtyMinutesAgo = new Date(
+      Date.now() - (global.feConfigs?.exportLimitMinutes || 0) * 60 * 1000
+    );
 
     // auth export times
     const authTimes = await User.findOne(
@@ -35,21 +39,27 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
     );
 
     if (!authTimes) {
-      throw new Error('上次导出未到半小时，每半小时仅可导出一次。');
+      const minutes = `${global.feConfigs?.exportLimitMinutes || 0} 分钟`;
+      throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`);
     }
 
-    // 统计数据
+    const where: any = [
-    const count = await PgClient.count(PgTrainingTableName, {
+      ['kb_id', kbId],
-      where: [['kb_id', kbId], 'AND', ['user_id', userId]]
+      'AND',
-    });
+      ['user_id', userId],
+      ...(fileId
+        ? fileId === OtherFileId
+          ? ["AND (file_id IS NULL OR file_id = '')"]
+          : ['AND', ['file_id', fileId]]
+        : [])
+    ];
     // 从 pg 中获取所有数据
     const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
       PgTrainingTableName,
       {
-        where: [['kb_id', kbId], 'AND', ['user_id', userId]],
+        where,
         fields: ['q', 'a', 'source'],
-        order: [{ field: 'id', mode: 'DESC' }],
+        order: [{ field: 'id', mode: 'DESC' }]
-        limit: count
       }
     );
 

client/src/pages/api/plugins/kb/data/getDataList.ts

@@ -58,7 +58,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         offset: pageSize * (pageNum - 1)
       }),
       PgClient.count(PgTrainingTableName, {
-        fields: ['kb_id'],
+        fields: ['id'],
         where
       })
     ]);

client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts

@@ -33,7 +33,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
         return {
           id: file._id,
           chunkLength: await PgClient.count(PgTrainingTableName, {
-            fields: ['kb_id'],
+            fields: ['id'],
             where: [
               ['user_id', userId],
               'AND',

client/src/pages/api/plugins/kb/file/list.ts

@@ -36,7 +36,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
           ? FileStatusEnum.embedding
           : FileStatusEnum.ready,
         chunkLength: await PgClient.count(PgTrainingTableName, {
-          fields: ['kb_id'],
+          fields: ['id'],
           where: [
             ['user_id', userId],
             'AND',
@@ -59,7 +59,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
             ? FileStatusEnum.embedding
             : FileStatusEnum.ready,
           chunkLength: await PgClient.count(PgTrainingTableName, {
-            fields: ['kb_id'],
+            fields: ['id'],
             where: [
               ['user_id', userId],
               'AND',

client/src/pages/api/system/getInitData.ts

@@ -1,4 +1,4 @@
-import type { FeConfigsType } from '@/types';
+import type { FeConfigsType, SystemEnvType } from '@/types';
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { readFileSync } from 'fs';
@@ -29,12 +29,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
   });
 }
 
-const defaultSystemEnv = {
+const defaultSystemEnv: SystemEnvType = {
   vectorMaxProcess: 15,
   qaMaxProcess: 15,
   pgIvfflatProbe: 20
 };
-const defaultFeConfigs = {
+const defaultFeConfigs: FeConfigsType = {
   show_emptyChat: true,
   show_register: false,
   show_appStore: false,
@@ -44,7 +44,7 @@ const defaultFeConfigs = {
   show_doc: true,
   systemTitle: 'FastGPT',
   authorText: 'Made by FastGPT Team.',
-  gitLoginKey: '',
+  exportLimitMinutes: 0,
   scripts: []
 };
 const defaultChatModels = [
@@ -99,8 +99,10 @@ export async function getInitConfig() {
     const res = JSON.parse(readFileSync(filename, 'utf-8'));
     console.log(res);
 
-    global.systemEnv = res.SystemParams || defaultSystemEnv;
+    global.systemEnv = res.SystemParams
-    global.feConfigs = res.FeConfig || defaultFeConfigs;
+      ? { ...defaultSystemEnv, ...res.SystemParams }
+      : defaultSystemEnv;
+    global.feConfigs = res.FeConfig ? { ...defaultFeConfigs, ...res.FeConfig } : defaultFeConfigs;
     global.chatModels = res.ChatModels || defaultChatModels;
     global.qaModel = res.QAModel || defaultQAModel;
     global.vectorModels = res.VectorModels || defaultVectorModels;

client/src/pages/kb/detail/components/DataCard.tsx

@@ -25,6 +25,7 @@ import MyTooltip from '@/components/MyTooltip';
 import MyInput from '@/components/MyInput';
 import { fileImgs } from '@/constants/common';
 import { useRequest } from '@/hooks/useRequest';
+import { feConfigs } from '@/store/static';
 
 const DataCard = ({ kbId }: { kbId: string }) => {
   const BoxRef = useRef<HTMLDivElement>(null);
@@ -80,24 +81,6 @@ const DataCard = ({ kbId }: { kbId: string }) => {
     [getData, pageNum, refetchTrainingData]
   );
 
-  // get al data and export csv
-  const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({
-    mutationFn: () => getExportDataList(kbId),
-    onSuccess(res) {
-      const text = Papa.unparse({
-        fields: ['question', 'answer', 'source'],
-        data: res
-      });
-      fileDownload({
-        text,
-        type: 'text/csv',
-        filename: 'data.csv'
-      });
-    },
-    successToast: '导出成功，下次导出需要半小时后',
-    errorToast: '导出异常'
-  });
-
   // get first page data
   const getFirstData = useCallback(
     debounce(() => {
@@ -121,6 +104,28 @@ const DataCard = ({ kbId }: { kbId: string }) => {
     [fileInfo?.filename]
   );
 
+  // get al data and export csv
+  const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({
+    mutationFn: () => getExportDataList({ kbId, fileId }),
+    onSuccess(res) {
+      const text = Papa.unparse({
+        fields: ['question', 'answer', 'source'],
+        data: res
+      });
+
+      const filenameSplit = fileInfo?.filename?.split('.') || [];
+      const filename = filenameSplit?.length <= 1 ? 'data' : filenameSplit.slice(0, -1).join('.');
+
+      fileDownload({
+        text,
+        type: 'text/csv',
+        filename
+      });
+    },
+    successToast: `导出成功，下次导出需要 ${feConfigs.exportLimitMinutes} 分钟后`,
+    errorToast: '导出异常'
+  });
+
   return (
     <Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
       <Flex alignItems={'center'}>
@@ -141,7 +146,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
           borderColor={'myBlue.600'}
           color={'myBlue.600'}
           isLoading={isLoadingExport || isLoading}
-          title={'半小时仅能导出1次'}
+          title={`${feConfigs} 分钟能导出 1 次`}
           onClick={onclickExport}
         >
           {t('dataset.Export')}

client/src/pages/kb/detail/components/Import/FileSelect.tsx

@@ -25,7 +25,7 @@ const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
 const CreateFileModal = dynamic(() => import('./CreateFileModal'));
 
 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
-const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`;
+const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;
 
 export type FileItemType = {
   id: string;

client/src/service/pg.ts

@@ -108,6 +108,7 @@ class Pg {
       }
       LIMIT ${props.limit || 10} OFFSET ${props.offset || 0}
     `;
+
     const pg = await connectPg();
     return pg.query<T>(sql);
   }

client/src/types/index.d.ts

@@ -28,6 +28,7 @@ export type FeConfigsType = {
   beianText?: string;
   googleClientVerKey?: string;
   gitLoginKey?: string;
+  exportLimitMinutes?: number;
   scripts?: { [key: string]: string }[];
 };
 export type SystemEnvType = {