diff --git a/client/data/config.json b/client/data/config.json index 5bfd65fc1..8ebc01f4e 100644 --- a/client/data/config.json +++ b/client/data/config.json @@ -9,7 +9,7 @@ "show_doc": true, "systemTitle": "FastGPT", "authorText": "Made by FastGPT Team.", - "gitLoginKey": "", + "exportLimitMinutes": 0, "scripts": [] }, "SystemParams": { @@ -61,4 +61,4 @@ "maxToken": 16000, "price": 0 } -} \ No newline at end of file +} diff --git a/client/src/api/plugins/kb.ts b/client/src/api/plugins/kb.ts index 7c69a9ec5..7e93698ce 100644 --- a/client/src/api/plugins/kb.ts +++ b/client/src/api/plugins/kb.ts @@ -53,14 +53,10 @@ export const getKbDataList = (data: GetKbDataListProps) => /** * 获取导出数据（不分页） */ -export const getExportDataList = (kbId: string) => - GET<[string, string, string][]>( - `/plugins/kb/data/exportModelData`, - { kbId }, - { - timeout: 600000 - } - ); +export const getExportDataList = (data: { kbId: string; fileId: string }) => + GET<[string, string, string][]>(`/plugins/kb/data/exportModelData`, data, { + timeout: 600000 + }); /** * 获取模型正在拆分数据的数量 diff --git a/client/src/pages/api/plugins/kb/data/exportModelData.ts b/client/src/pages/api/plugins/kb/data/exportModelData.ts index 9b3bc90a4..ba4b1b7d9 100644 --- a/client/src/pages/api/plugins/kb/data/exportModelData.ts +++ b/client/src/pages/api/plugins/kb/data/exportModelData.ts @@ -4,11 +4,13 @@ import { connectToDatabase, User } from '@/service/mongo'; import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; import { PgTrainingTableName } from '@/constants/plugin'; +import { OtherFileId } from '@/constants/kb'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { - let { kbId } = req.query as { + let { kbId, fileId } = req.query as { kbId: string; + fileId: string; }; if (!kbId) { @@ -20,7 +22,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // 凭证校验 const { userId } = await authUser({ req, authToken: true }); - const thirtyMinutesAgo = new Date(Date.now() - 30 * 60 * 1000); + const thirtyMinutesAgo = new Date( + Date.now() - (global.feConfigs?.exportLimitMinutes || 0) * 60 * 1000 + ); // auth export times const authTimes = await User.findOne( @@ -35,21 +39,27 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ); if (!authTimes) { - throw new Error('上次导出未到半小时，每半小时仅可导出一次。'); + const minutes = `${global.feConfigs?.exportLimitMinutes || 0} 分钟`; + throw new Error(`上次导出未到 ${minutes}，每 ${minutes}仅可导出一次。`); } - // 统计数据 - const count = await PgClient.count(PgTrainingTableName, { - where: [['kb_id', kbId], 'AND', ['user_id', userId]] - }); + const where: any = [ + ['kb_id', kbId], + 'AND', + ['user_id', userId], + ...(fileId + ? fileId === OtherFileId + ? ["AND (file_id IS NULL OR file_id = '')"] + : ['AND', ['file_id', fileId]] + : []) + ]; // 从 pg 中获取所有数据 const pgData = await PgClient.select<{ q: string; a: string; source: string }>( PgTrainingTableName, { - where: [['kb_id', kbId], 'AND', ['user_id', userId]], + where, fields: ['q', 'a', 'source'], - order: [{ field: 'id', mode: 'DESC' }], - limit: count + order: [{ field: 'id', mode: 'DESC' }] } ); diff --git a/client/src/pages/api/plugins/kb/data/getDataList.ts b/client/src/pages/api/plugins/kb/data/getDataList.ts index 5d928a983..6c8fecf3f 100644 --- a/client/src/pages/api/plugins/kb/data/getDataList.ts +++ b/client/src/pages/api/plugins/kb/data/getDataList.ts @@ -58,7 +58,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< offset: pageSize * (pageNum - 1) }), PgClient.count(PgTrainingTableName, { - fields: ['kb_id'], + fields: ['id'], where }) ]); diff --git a/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts b/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts index 5dc52287b..1cb1c979e 100644 --- a/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts +++ b/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts @@ -33,7 +33,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< return { id: file._id, chunkLength: await PgClient.count(PgTrainingTableName, { - fields: ['kb_id'], + fields: ['id'], where: [ ['user_id', userId], 'AND', diff --git a/client/src/pages/api/plugins/kb/file/list.ts b/client/src/pages/api/plugins/kb/file/list.ts index 234ac55bb..2577213e4 100644 --- a/client/src/pages/api/plugins/kb/file/list.ts +++ b/client/src/pages/api/plugins/kb/file/list.ts @@ -36,7 +36,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ? FileStatusEnum.embedding : FileStatusEnum.ready, chunkLength: await PgClient.count(PgTrainingTableName, { - fields: ['kb_id'], + fields: ['id'], where: [ ['user_id', userId], 'AND', @@ -59,7 +59,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ? FileStatusEnum.embedding : FileStatusEnum.ready, chunkLength: await PgClient.count(PgTrainingTableName, { - fields: ['kb_id'], + fields: ['id'], where: [ ['user_id', userId], 'AND', diff --git a/client/src/pages/api/system/getInitData.ts b/client/src/pages/api/system/getInitData.ts index a50eb4810..ca3f584bc 100644 --- a/client/src/pages/api/system/getInitData.ts +++ b/client/src/pages/api/system/getInitData.ts @@ -1,4 +1,4 @@ -import type { FeConfigsType } from '@/types'; +import type { FeConfigsType, SystemEnvType } from '@/types'; import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@/service/response'; import { readFileSync } from 'fs'; @@ -29,12 +29,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) }); } -const defaultSystemEnv = { +const defaultSystemEnv: SystemEnvType = { vectorMaxProcess: 15, qaMaxProcess: 15, pgIvfflatProbe: 20 }; -const defaultFeConfigs = { +const defaultFeConfigs: FeConfigsType = { show_emptyChat: true, show_register: false, show_appStore: false, @@ -44,7 +44,7 @@ const defaultFeConfigs = { show_doc: true, systemTitle: 'FastGPT', authorText: 'Made by FastGPT Team.', - gitLoginKey: '', + exportLimitMinutes: 0, scripts: [] }; const defaultChatModels = [ @@ -99,8 +99,10 @@ export async function getInitConfig() { const res = JSON.parse(readFileSync(filename, 'utf-8')); console.log(res); - global.systemEnv = res.SystemParams || defaultSystemEnv; - global.feConfigs = res.FeConfig || defaultFeConfigs; + global.systemEnv = res.SystemParams + ? { ...defaultSystemEnv, ...res.SystemParams } + : defaultSystemEnv; + global.feConfigs = res.FeConfig ? { ...defaultFeConfigs, ...res.FeConfig } : defaultFeConfigs; global.chatModels = res.ChatModels || defaultChatModels; global.qaModel = res.QAModel || defaultQAModel; global.vectorModels = res.VectorModels || defaultVectorModels; diff --git a/client/src/pages/kb/detail/components/DataCard.tsx b/client/src/pages/kb/detail/components/DataCard.tsx index 3d95e74fb..465b80656 100644 --- a/client/src/pages/kb/detail/components/DataCard.tsx +++ b/client/src/pages/kb/detail/components/DataCard.tsx @@ -25,6 +25,7 @@ import MyTooltip from '@/components/MyTooltip'; import MyInput from '@/components/MyInput'; import { fileImgs } from '@/constants/common'; import { useRequest } from '@/hooks/useRequest'; +import { feConfigs } from '@/store/static'; const DataCard = ({ kbId }: { kbId: string }) => { const BoxRef = useRef(null); @@ -80,24 +81,6 @@ const DataCard = ({ kbId }: { kbId: string }) => { [getData, pageNum, refetchTrainingData] ); - // get al data and export csv - const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({ - mutationFn: () => getExportDataList(kbId), - onSuccess(res) { - const text = Papa.unparse({ - fields: ['question', 'answer', 'source'], - data: res - }); - fileDownload({ - text, - type: 'text/csv', - filename: 'data.csv' - }); - }, - successToast: '导出成功，下次导出需要半小时后', - errorToast: '导出异常' - }); - // get first page data const getFirstData = useCallback( debounce(() => { @@ -121,6 +104,28 @@ const DataCard = ({ kbId }: { kbId: string }) => { [fileInfo?.filename] ); + // get al data and export csv + const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({ + mutationFn: () => getExportDataList({ kbId, fileId }), + onSuccess(res) { + const text = Papa.unparse({ + fields: ['question', 'answer', 'source'], + data: res + }); + + const filenameSplit = fileInfo?.filename?.split('.') || []; + const filename = filenameSplit?.length <= 1 ? 'data' : filenameSplit.slice(0, -1).join('.'); + + fileDownload({ + text, + type: 'text/csv', + filename + }); + }, + successToast: `导出成功，下次导出需要 ${feConfigs.exportLimitMinutes} 分钟后`, + errorToast: '导出异常' + }); + return ( @@ -141,7 +146,7 @@ const DataCard = ({ kbId }: { kbId: string }) => { borderColor={'myBlue.600'} color={'myBlue.600'} isLoading={isLoadingExport || isLoading} - title={'半小时仅能导出1次'} + title={`${feConfigs} 分钟能导出 1 次`} onClick={onclickExport} > {t('dataset.Export')} diff --git a/client/src/pages/kb/detail/components/Import/FileSelect.tsx b/client/src/pages/kb/detail/components/Import/FileSelect.tsx index b1d31a11a..ea95b8582 100644 --- a/client/src/pages/kb/detail/components/Import/FileSelect.tsx +++ b/client/src/pages/kb/detail/components/Import/FileSelect.tsx @@ -25,7 +25,7 @@ const UrlFetchModal = dynamic(() => import('./UrlFetchModal')); const CreateFileModal = dynamic(() => import('./CreateFileModal')); const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); -const csvTemplate = `question,answer,source\n"什么是 laf","laf 是一个云函数开发平台……","laf git doc"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……","sealos git doc"`; +const csvTemplate = `question,answer\n"什么是 laf","laf 是一个云函数开发平台……"\n"什么是 sealos","Sealos 是以 kubernetes 为内核的云操作系统发行版,可以……"`; export type FileItemType = { id: string; diff --git a/client/src/service/pg.ts b/client/src/service/pg.ts index 1540268a8..edffeb443 100644 --- a/client/src/service/pg.ts +++ b/client/src/service/pg.ts @@ -108,6 +108,7 @@ class Pg { } LIMIT ${props.limit || 10} OFFSET ${props.offset || 0} `; + const pg = await connectPg(); return pg.query(sql); } diff --git a/client/src/types/index.d.ts b/client/src/types/index.d.ts index 8e32f4dc4..ede3ae52a 100644 --- a/client/src/types/index.d.ts +++ b/client/src/types/index.d.ts @@ -28,6 +28,7 @@ export type FeConfigsType = { beianText?: string; googleClientVerKey?: string; gitLoginKey?: string; + exportLimitMinutes?: number; scripts?: { [key: string]: string }[]; }; export type SystemEnvType = {