export data limit file_id

2023-09-10 16:25:52 +08:00
parent d8f660370f
commit a85e0e9e2e
11 changed files with 65 additions and 50 deletions
--- a/client/data/config.json
+++ b/client/data/config.json
@@ -9,7 +9,7 @@
    "show_doc": true,
    "systemTitle": "FastGPT",
    "authorText": "Made by FastGPT Team.",
-    "gitLoginKey": "",
+    "exportLimitMinutes": 0,
    "scripts": []
  },
  "SystemParams": {
@@ -61,4 +61,4 @@
    "maxToken": 16000,
    "price": 0
  }
-}
+}
--- a/client/src/api/plugins/kb.ts
+++ b/client/src/api/plugins/kb.ts
@@ -53,14 +53,10 @@ export const getKbDataList = (data: GetKbDataListProps) =>
 /**
 * 获取导出数据（不分页）
 */
-export const getExportDataList = (kbId: string) =>
-  GET<[string, string, string][]>(
-    `/plugins/kb/data/exportModelData`,
-    { kbId },
-    {
-      timeout: 600000
-    }
-  );
+export const getExportDataList = (data: { kbId: string; fileId: string }) =>
+  GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, {
+    timeout: 600000
+  });

 /**
 * 获取模型正在拆分数据的数量
--- a/client/src/pages/api/plugins/kb/data/exportModelData.ts
+++ b/client/src/pages/api/plugins/kb/data/exportModelData.ts
@@ -4,11 +4,13 @@ import { connectToDatabase, User } from '@/service/mongo';
 import { authUser } from '@/service/utils/auth';
 import { PgClient } from '@/service/pg';
 import { PgTrainingTableName } from '@/constants/plugin';
+import { OtherFileId } from '@/constants/kb';

 export default async function handler(req: NextApiRequest, res: NextApiResponse<any>) {
  try {
-    let { kbId } = req.query as {
+    let { kbId, fileId } = req.query as {
      kbId: string;
+      fileId: string;
    };

    if (!kbId) {
@@ -20,7 +22,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    // 凭证校验
    const { userId } = await authUser({ req, authToken: true });

-    const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000);
+    const thirtyMinutesAgo = new Date(
+      Date.now() - (global.feConfigs?.exportLimitMinutes || 0) * 60 * 1000
+    );

    // auth export times
    const authTimes = await User.findOne(
@@ -35,21 +39,27 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
    );

    if (!authTimes) {
-      throw new Error('上次导出未到半小时，每半小时仅可导出一次。');
+      const minutes = `${global.feConfigs?.exportLimitMinutes || 0} 分钟`;
+      throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`);
    }

-    // 统计数据
-    const count = await PgClient.count(PgTrainingTableName, {
-      where: [['kb_id', kbId], 'AND', ['user_id', userId]]
-    });
+    const where: any = [
+      ['kb_id', kbId],
+      'AND',
+      ['user_id', userId],
+      ...(fileId
+        ? fileId === OtherFileId
+          ? ["AND (file_id IS NULL OR file_id = '')"]
+          : ['AND', ['file_id', fileId]]
+        : [])
+    ];
    // 从 pg 中获取所有数据
    const pgData = await PgClient.select<{ q: string; a: string; source: string }>(
      PgTrainingTableName,
      {
-        where: [['kb_id', kbId], 'AND', ['user_id', userId]],
+        where,
        fields: ['q', 'a', 'source'],
-        order: [{ field: 'id', mode: 'DESC' }],
-        limit: count
+        order: [{ field: 'id', mode: 'DESC' }]
      }
    );

--- a/client/src/pages/api/plugins/kb/data/getDataList.ts
+++ b/client/src/pages/api/plugins/kb/data/getDataList.ts
@@ -58,7 +58,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
        offset: pageSize * (pageNum - 1)
      }),
      PgClient.count(PgTrainingTableName, {
-        fields: ['kb_id'],
+        fields: ['id'],
        where
      })
    ]);
--- a/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts
+++ b/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts
@@ -33,7 +33,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
        return {
          id: file._id,
          chunkLength: await PgClient.count(PgTrainingTableName, {
-            fields: ['kb_id'],
+            fields: ['id'],
            where: [
              ['user_id', userId],
              'AND',
--- a/client/src/pages/api/plugins/kb/file/list.ts
+++ b/client/src/pages/api/plugins/kb/file/list.ts
@@ -36,7 +36,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
          ? FileStatusEnum.embedding
          : FileStatusEnum.ready,
        chunkLength: await PgClient.count(PgTrainingTableName, {
-          fields: ['kb_id'],
+          fields: ['id'],
          where: [
            ['user_id', userId],
            'AND',
@@ -59,7 +59,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse<
            ? FileStatusEnum.embedding
            : FileStatusEnum.ready,
          chunkLength: await PgClient.count(PgTrainingTableName, {
-            fields: ['kb_id'],
+            fields: ['id'],
            where: [
              ['user_id', userId],
              'AND',
--- a/client/src/pages/api/system/getInitData.ts
+++ b/client/src/pages/api/system/getInitData.ts
@@ -1,4 +1,4 @@
-import type { FeConfigsType } from '@/types';
+import type { FeConfigsType, SystemEnvType } from '@/types';
 import type { NextApiRequest, NextApiResponse } from 'next';
 import { jsonRes } from '@/service/response';
 import { readFileSync } from 'fs';
@@ -29,12 +29,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
  });
 }

-const defaultSystemEnv = {
+const defaultSystemEnv: SystemEnvType = {
  vectorMaxProcess: 15,
  qaMaxProcess: 15,
  pgIvfflatProbe: 20
 };
-const defaultFeConfigs = {
+const defaultFeConfigs: FeConfigsType = {
  show_emptyChat: true,
  show_register: false,
  show_appStore: false,
@@ -44,7 +44,7 @@ const defaultFeConfigs = {
  show_doc: true,
  systemTitle: 'FastGPT',
  authorText: 'Made by FastGPT Team.',
-  gitLoginKey: '',
+  exportLimitMinutes: 0,
  scripts: []
 };
 const defaultChatModels = [
@@ -99,8 +99,10 @@ export async function getInitConfig() {
    const res = JSON.parse(readFileSync(filename, 'utf-8'));
    console.log(res);

-    global.systemEnv = res.SystemParams || defaultSystemEnv;
-    global.feConfigs = res.FeConfig || defaultFeConfigs;
+    global.systemEnv = res.SystemParams
+      ? { ...defaultSystemEnv, ...res.SystemParams }
+      : defaultSystemEnv;
+    global.feConfigs = res.FeConfig ? { ...defaultFeConfigs, ...res.FeConfig } : defaultFeConfigs;
    global.chatModels = res.ChatModels || defaultChatModels;
    global.qaModel = res.QAModel || defaultQAModel;
    global.vectorModels = res.VectorModels || defaultVectorModels;
--- a/client/src/pages/kb/detail/components/DataCard.tsx
+++ b/client/src/pages/kb/detail/components/DataCard.tsx
@@ -25,6 +25,7 @@ import MyTooltip from '@/components/MyTooltip';
 import MyInput from '@/components/MyInput';
 import { fileImgs } from '@/constants/common';
 import { useRequest } from '@/hooks/useRequest';
+import { feConfigs } from '@/store/static';

 const DataCard = ({ kbId }: { kbId: string }) => {
  const BoxRef = useRef<HTMLDivElement>(null);
@@ -80,24 +81,6 @@ const DataCard = ({ kbId }: { kbId: string }) => {
    [getData, pageNum, refetchTrainingData]
  );

-  // get al data and export csv
-  const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({
-    mutationFn: () => getExportDataList(kbId),
-    onSuccess(res) {
-      const text = Papa.unparse({
-        fields: ['question', 'answer', 'source'],
-        data: res
-      });
-      fileDownload({
-        text,
-        type: 'text/csv',
-        filename: 'data.csv'
-      });
-    },
-    successToast: '导出成功，下次导出需要半小时后',
-    errorToast: '导出异常'
-  });
-
  // get first page data
  const getFirstData = useCallback(
    debounce(() => {
@@ -121,6 +104,28 @@ const DataCard = ({ kbId }: { kbId: string }) => {
    [fileInfo?.filename]
  );

+  // get al data and export csv
+  const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({
+    mutationFn: () => getExportDataList({ kbId, fileId }),
+    onSuccess(res) {
+      const text = Papa.unparse({
+        fields: ['question', 'answer', 'source'],
+        data: res
+      });
+
+      const filenameSplit = fileInfo?.filename?.split('.') || [];
+      const filename = filenameSplit?.length <= 1 ? 'data' : filenameSplit.slice(0, -1).join('.');
+
+      fileDownload({
+        text,
+        type: 'text/csv',
+        filename
+      });
+    },
+    successToast: `导出成功，下次导出需要 ${feConfigs.exportLimitMinutes} 分钟后`,
+    errorToast: '导出异常'
+  });
+
  return (
    <Box ref={BoxRef} position={'relative'} px={5} py={[1, 5]} h={'100%'} overflow={'overlay'}>
      <Flex alignItems={'center'}>
@@ -141,7 +146,7 @@ const DataCard = ({ kbId }: { kbId: string }) => {
          borderColor={'myBlue.600'}
          color={'myBlue.600'}
          isLoading={isLoadingExport || isLoading}
-          title={'半小时仅能导出1次'}
+          title={`${feConfigs} 分钟能导出 1 次`}
          onClick={onclickExport}
        >
          {t('dataset.Export')}
--- a/client/src/pages/kb/detail/components/Import/FileSelect.tsx
+++ b/client/src/pages/kb/detail/components/Import/FileSelect.tsx
@@ -25,7 +25,7 @@ const UrlFetchModal = dynamic(() => import('./UrlFetchModal'));
 const CreateFileModal = dynamic(() => import('./CreateFileModal'));

 const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12);
-const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`;
+const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`;

 export type FileItemType = {
  id: string;
--- a/client/src/service/pg.ts
+++ b/client/src/service/pg.ts
@@ -108,6 +108,7 @@ class Pg {
      }
      LIMIT ${props.limit || 10} OFFSET ${props.offset || 0}
    `;
+
    const pg = await connectPg();
    return pg.query<T>(sql);
  }
--- a/client/src/types/index.d.ts
+++ b/client/src/types/index.d.ts
@@ -28,6 +28,7 @@ export type FeConfigsType = {
  beianText?: string;
  googleClientVerKey?: string;
  gitLoginKey?: string;
+  exportLimitMinutes?: number;
  scripts?: { [key: string]: string }[];
 };
 export type SystemEnvType = {