From ba6c2d27d5fe7deaa7955ade41a240bfaccf37aa Mon Sep 17 00:00:00 2001 From: archer <545436317@qq.com> Date: Sun, 10 Sep 2023 11:29:36 +0800 Subject: [PATCH] feat: fileCard and dataCard --- client/public/imgs/files/file.svg | 1 + client/public/locales/en/common.json | 13 ++ client/public/locales/zh/common.json | 13 ++ client/src/api/plugins/kb.ts | 26 ++- client/src/api/request/kb.d.ts | 8 + .../components/Icon/icons/light/search.svg | 8 + client/src/components/Icon/index.tsx | 3 +- client/src/components/MyInput/index.tsx | 28 +++ client/src/constants/common.ts | 3 +- client/src/constants/kb.ts | 5 + .../pages/api/plugins/kb/data/getDataList.ts | 12 +- .../api/plugins/kb/file/delFileByFileId.ts | 55 +++++ .../api/plugins/kb/file/deleteEmptyFiles.ts | 59 ++++++ .../pages/api/plugins/kb/file/getFileInfo.ts | 43 ++++ client/src/pages/api/plugins/kb/file/list.ts | 82 ++++++++ .../pages/kb/detail/components/DataCard.tsx | 115 +++++----- .../pages/kb/detail/components/FileCard.tsx | 196 ++++++++++++++++++ .../kb/detail/components/Import/Chunk.tsx | 2 +- .../pages/kb/detail/components/Import/Csv.tsx | 2 +- .../pages/kb/detail/components/Import/QA.tsx | 2 +- .../kb/detail/components/InputDataModal.tsx | 1 + client/src/pages/kb/detail/index.tsx | 31 ++- client/src/service/events/generateQA.ts | 2 +- client/src/service/events/generateVector.ts | 4 +- client/src/service/lib/gridfs.ts | 10 +- client/src/service/utils/auth.ts | 2 +- client/src/types/plugin.d.ts | 19 ++ client/src/utils/tools.ts | 13 ++ 28 files changed, 675 insertions(+), 83 deletions(-) create mode 100644 client/public/imgs/files/file.svg create mode 100644 client/src/components/Icon/icons/light/search.svg create mode 100644 client/src/components/MyInput/index.tsx create mode 100644 client/src/pages/api/plugins/kb/file/delFileByFileId.ts create mode 100644 client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts create mode 100644 client/src/pages/api/plugins/kb/file/getFileInfo.ts create mode 100644 client/src/pages/api/plugins/kb/file/list.ts create mode 100644 client/src/pages/kb/detail/components/FileCard.tsx diff --git a/client/public/imgs/files/file.svg b/client/public/imgs/files/file.svg new file mode 100644 index 000000000..f1b07130b --- /dev/null +++ b/client/public/imgs/files/file.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/client/public/locales/en/common.json b/client/public/locales/en/common.json index ebfffff6d..39fa7168f 100644 --- a/client/public/locales/en/common.json +++ b/client/public/locales/en/common.json @@ -78,6 +78,7 @@ "Copy Successful": "Copy Successful", "Course": "", "Delete": "Delete", + "Delete Failed": "Delete Failed", "Delete Success": "Delete Successful", "Delete Warning": "Warning", "Filed is repeat": "Filed is repeated", @@ -86,10 +87,13 @@ "Output": "Output", "Password inconsistency": "Password inconsistency", "Rename": "Rename", + "Search": "Search", + "Status": "Status", "export": "" }, "dataset": { "Confirm to delete the data": "Confirm to delete the data?", + "Export": "Export", "Queue Desc": "This data refers to the current amount of training for the entire system. FastGPT uses queued training, and if you have too much data to train, you may need to wait for a while", "System Data Queue": "Data Queue" }, @@ -99,9 +103,11 @@ "Create File": "Create File", "Create file": "Create file", "Drag and drop": "Drag and drop files here", + "Embedding": "Embedding", "Fetch Url": "Fetch Url", "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "If the imported file is garbled, please convert CSV to UTF-8 encoding format", "Parse": "{{name}} Parsing...", + "Ready": "Ready", "Release the mouse to upload the file": "Release the mouse to upload the file", "Select a maximum of 10 files": "Select a maximum of 10 files", "Uploading": "Uploading: {{name}}, Progress: {{percent}}%", @@ -156,11 +162,18 @@ "slogan": "Let the AI know more about you" }, "kb": { + "Chunk Length": "Chunk Length", + "Confirm to delete the file": "Are you sure to delete the file and all its data?", "Create Folder": "Create Folder", "Delete Dataset Error": "Delete dataset failed", "Edit Folder": "Edit Folder", + "File Size": "File Size", + "Filename": "Filename", + "Files": "{{total}} Files", "Folder Name": "Input folder name", "My Dataset": "My Dataset", + "Other Data": "Other Data", + "Upload Time": "Upload Time", "deleteDatasetTips": "Are you sure to delete the knowledge base? Data cannot be recovered after deletion, please confirm!", "deleteFolderTips": "Are you sure to delete this folder and all the knowledge bases it contains? Data cannot be recovered after deletion, please confirm!" }, diff --git a/client/public/locales/zh/common.json b/client/public/locales/zh/common.json index 22e239d59..7554e62c8 100644 --- a/client/public/locales/zh/common.json +++ b/client/public/locales/zh/common.json @@ -78,6 +78,7 @@ "Copy Successful": "复制成功", "Course": "", "Delete": "删除", + "Delete Failed": "删除失败", "Delete Success": "删除成功", "Delete Warning": "删除警告", "Filed is repeat": "", @@ -86,10 +87,13 @@ "Output": "输出", "Password inconsistency": "两次密码不一致", "Rename": "重命名", + "Search": "搜索", + "Status": "状态", "export": "" }, "dataset": { "Confirm to delete the data": "确认删除该数据？", + "Export": "导出", "Queue Desc": "该数据是指整个系统当前待训练的数量。{{title}} 采用排队训练的方式，如果待训练的数据过多，可能需要等待一段时间", "System Data Queue": "排队长度" }, @@ -99,9 +103,11 @@ "Create File": "创建新文件", "Create file": "创建文件", "Drag and drop": "拖拽文件至此", + "Embedding": "索引中", "Fetch Url": "链接读取", "If the imported file is garbled, please convert CSV to UTF-8 encoding format": "如果导入文件乱码，请将 CSV 转成 UTF-8 编码格式", "Parse": "{{name}} 解析中...", + "Ready": "可用", "Release the mouse to upload the file": "松开鼠标上传文件", "Select a maximum of 10 files": "最多选择10个文件", "Uploading": "正在上传 {{name}}，进度: {{percent}}%", @@ -156,11 +162,18 @@ "slogan": "让 AI 更懂你的知识" }, "kb": { + "Chunk Length": "数据总量", + "Confirm to delete the file": "确认删除该文件及其所有数据？", "Create Folder": "创建文件夹", "Delete Dataset Error": "删除知识库异常", "Edit Folder": "编辑文件夹", + "File Size": "文件大小", + "Filename": "文件名", + "Files": "文件: {{total}}个", "Folder Name": "输入文件夹名称", "My Dataset": "我的知识库", + "Other Data": "其他数据", + "Upload Time": "上传时间", "deleteDatasetTips": "确认删除该知识库？删除后数据无法恢复，请确认！", "deleteFolderTips": "确认删除该文件夹及其包含的所有知识库？删除后数据无法恢复，请确认！" }, diff --git a/client/src/api/plugins/kb.ts b/client/src/api/plugins/kb.ts index 270bbdbc9..57fffa79b 100644 --- a/client/src/api/plugins/kb.ts +++ b/client/src/api/plugins/kb.ts @@ -1,6 +1,12 @@ import { GET, POST, PUT, DELETE } from '../request'; -import type { DatasetItemType, KbItemType, KbListItemType, KbPathItemType } from '@/types/plugin'; -import { RequestPaging } from '@/types/index'; +import type { + DatasetItemType, + FileInfo, + KbFileItemType, + KbItemType, + KbListItemType, + KbPathItemType +} from '@/types/plugin'; import { TrainingModeEnum } from '@/constants/plugin'; import { Props as PushDataProps, @@ -11,7 +17,7 @@ import { Response as SearchTestResponse } from '@/pages/api/openapi/kb/searchTest'; import { Props as UpdateDataProps } from '@/pages/api/openapi/kb/updateData'; -import type { KbUpdateParams, CreateKbParams } from '../request/kb'; +import type { KbUpdateParams, CreateKbParams, GetKbDataListProps } from '../request/kb'; import { QuoteItemType } from '@/types/chat'; /* knowledge base */ @@ -29,11 +35,17 @@ export const putKbById = (data: KbUpdateParams) => PUT(`/plugins/kb/update`, dat export const delKbById = (id: string) => DELETE(`/plugins/kb/delete?id=${id}`); +/* kb file */ +export const getKbFiles = (kbId: string) => + GET(`/plugins/kb/file/list`, { kbId }); +export const deleteKbFileById = (params: { fileId: string; kbId: string }) => + DELETE(`/plugins/kb/file/delFileByFileId`, params); +export const getFileInfoById = (fileId: string) => + GET(`/plugins/kb/file/getFileInfo`, { fileId }); +export const delEmptyFiles = (kbId: string) => + DELETE(`/plugins/kb/file/deleteEmptyFiles`, { kbId }); + /* kb data */ -type GetKbDataListProps = RequestPaging & { - kbId: string; - searchText: string; -}; export const getKbDataList = (data: GetKbDataListProps) => POST(`/plugins/kb/data/getDataList`, data); diff --git a/client/src/api/request/kb.d.ts b/client/src/api/request/kb.d.ts index e139ca668..a9d698394 100644 --- a/client/src/api/request/kb.d.ts +++ b/client/src/api/request/kb.d.ts @@ -1,4 +1,6 @@ import { KbTypeEnum } from '@/constants/kb'; +import type { RequestPaging } from '@/types'; + export type KbUpdateParams = { id: string; tags?: string; @@ -13,3 +15,9 @@ export type CreateKbParams = { vectorModel?: string; type: `${KbTypeEnum}`; }; + +export type GetKbDataListProps = RequestPaging & { + kbId: string; + searchText: string; + fileId: string; +}; diff --git a/client/src/components/Icon/icons/light/search.svg b/client/src/components/Icon/icons/light/search.svg new file mode 100644 index 000000000..493230bee --- /dev/null +++ b/client/src/components/Icon/icons/light/search.svg @@ -0,0 +1,8 @@ + + + + \ No newline at end of file diff --git a/client/src/components/Icon/index.tsx b/client/src/components/Icon/index.tsx index e004f304f..3f6cb75a4 100644 --- a/client/src/components/Icon/index.tsx +++ b/client/src/components/Icon/index.tsx @@ -81,7 +81,8 @@ const map = { badLight: require('./icons/light/bad.svg').default, markLight: require('./icons/light/mark.svg').default, retryLight: require('./icons/light/retry.svg').default, - rightArrowLight: require('./icons/light/rightArrow.svg').default + rightArrowLight: require('./icons/light/rightArrow.svg').default, + searchLight: require('./icons/light/search.svg').default }; export type IconName = keyof typeof map; diff --git a/client/src/components/MyInput/index.tsx b/client/src/components/MyInput/index.tsx new file mode 100644 index 000000000..30e9d6b52 --- /dev/null +++ b/client/src/components/MyInput/index.tsx @@ -0,0 +1,28 @@ +import React from 'react'; +import { Flex, Input, InputProps } from '@chakra-ui/react'; + +interface Props extends InputProps { + leftIcon?: React.ReactNode; +} + +const MyInput = ({ leftIcon, ...props }: Props) => { + return ( + + + {leftIcon && ( + + {leftIcon} + + )} + + ); +}; + +export default MyInput; diff --git a/client/src/constants/common.ts b/client/src/constants/common.ts index 0604a33f1..103fa9748 100644 --- a/client/src/constants/common.ts +++ b/client/src/constants/common.ts @@ -10,7 +10,8 @@ export const fileImgs = [ { suffix: 'csv', src: '/imgs/files/csv.svg' }, { suffix: '(doc|docs)', src: '/imgs/files/doc.svg' }, { suffix: 'txt', src: '/imgs/files/txt.svg' }, - { suffix: 'md', src: '/imgs/files/markdown.svg' } + { suffix: 'md', src: '/imgs/files/markdown.svg' }, + { suffix: '.', src: '/imgs/files/file.svg' } ]; export enum TrackEventName { diff --git a/client/src/constants/kb.ts b/client/src/constants/kb.ts index 47742a1a5..104cb589a 100644 --- a/client/src/constants/kb.ts +++ b/client/src/constants/kb.ts @@ -19,6 +19,10 @@ export enum KbTypeEnum { folder = 'folder', dataset = 'dataset' } +export enum FileStatusEnum { + embedding = 'embedding', + ready = 'ready' +} export const KbTypeMap = { [KbTypeEnum.folder]: { @@ -30,3 +34,4 @@ export const KbTypeMap = { }; export const FolderAvatarSrc = '/imgs/files/folder.svg'; +export const OtherFileId = 'other'; diff --git a/client/src/pages/api/plugins/kb/data/getDataList.ts b/client/src/pages/api/plugins/kb/data/getDataList.ts index cffbc45eb..5d928a983 100644 --- a/client/src/pages/api/plugins/kb/data/getDataList.ts +++ b/client/src/pages/api/plugins/kb/data/getDataList.ts @@ -5,6 +5,7 @@ import { authUser } from '@/service/utils/auth'; import { PgClient } from '@/service/pg'; import type { KbDataItemType } from '@/types/plugin'; import { PgTrainingTableName } from '@/constants/plugin'; +import { OtherFileId } from '@/constants/kb'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -12,12 +13,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< kbId, pageNum = 1, pageSize = 10, - searchText = '' + searchText = '', + fileId = '' } = req.body as { kbId: string; pageNum: number; pageSize: number; searchText: string; + fileId: string; }; if (!kbId) { throw new Error('缺少参数'); @@ -33,6 +36,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ['user_id', userId], 'AND', ['kb_id', kbId], + ...(fileId + ? fileId === OtherFileId + ? ["AND (file_id IS NULL OR file_id = '')"] + : ['AND', ['file_id', fileId]] + : []), ...(searchText ? [ 'AND', @@ -50,7 +58,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< offset: pageSize * (pageNum - 1) }), PgClient.count(PgTrainingTableName, { - fields: ['id'], + fields: ['kb_id'], where }) ]); diff --git a/client/src/pages/api/plugins/kb/file/delFileByFileId.ts b/client/src/pages/api/plugins/kb/file/delFileByFileId.ts new file mode 100644 index 000000000..0e6b7767c --- /dev/null +++ b/client/src/pages/api/plugins/kb/file/delFileByFileId.ts @@ -0,0 +1,55 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@/service/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authUser } from '@/service/utils/auth'; +import { GridFSStorage } from '@/service/lib/gridfs'; +import { PgClient } from '@/service/pg'; +import { PgTrainingTableName } from '@/constants/plugin'; +import { Types } from 'mongoose'; +import { OtherFileId } from '@/constants/kb'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + + const { fileId, kbId } = req.query as { fileId: string; kbId: string }; + + if (!fileId || !kbId) { + throw new Error('fileId and kbId is required'); + } + + // 凭证校验 + const { userId } = await authUser({ req, authToken: true }); + + if (fileId === OtherFileId) { + await PgClient.delete(PgTrainingTableName, { + where: [ + ['user_id', userId], + 'AND', + ['kb_id', kbId], + "AND (file_id IS NULL OR file_id = '')" + ] + }); + } else { + const gridFs = new GridFSStorage('dataset', userId); + const bucket = gridFs.GridFSBucket(); + + await gridFs.findAndAuthFile(fileId); + + // delete all pg data + await PgClient.delete(PgTrainingTableName, { + where: [['user_id', userId], 'AND', ['kb_id', kbId], 'AND', ['file_id', fileId]] + }); + + // delete file + await bucket.delete(new Types.ObjectId(fileId)); + } + + jsonRes(res); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts b/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts new file mode 100644 index 000000000..5dc52287b --- /dev/null +++ b/client/src/pages/api/plugins/kb/file/deleteEmptyFiles.ts @@ -0,0 +1,59 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@/service/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authUser } from '@/service/utils/auth'; +import { GridFSStorage } from '@/service/lib/gridfs'; +import { PgClient } from '@/service/pg'; +import { PgTrainingTableName } from '@/constants/plugin'; +import { Types } from 'mongoose'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + + const { kbId } = req.query as { kbId: string }; + // 凭证校验 + const { userId } = await authUser({ req, authToken: true }); + + const gridFs = new GridFSStorage('dataset', userId); + const bucket = gridFs.GridFSBucket(); + + const files = await bucket + // 1 hours expired + .find({ + uploadDate: { $lte: new Date(Date.now() - 60 * 1000) }, + ['metadata.kbId']: kbId, + ['metadata.userId']: userId + }) + .sort({ _id: -1 }) + .toArray(); + + const data = await Promise.all( + files.map(async (file) => { + return { + id: file._id, + chunkLength: await PgClient.count(PgTrainingTableName, { + fields: ['kb_id'], + where: [ + ['user_id', userId], + 'AND', + ['kb_id', kbId], + 'AND', + ['file_id', String(file._id)] + ] + }) + }; + }) + ); + + await Promise.all( + data + .filter((item) => item.chunkLength === 0) + .map((file) => bucket.delete(new Types.ObjectId(file.id))) + ); + + jsonRes(res); + } catch (err) { + jsonRes(res); + } +} diff --git a/client/src/pages/api/plugins/kb/file/getFileInfo.ts b/client/src/pages/api/plugins/kb/file/getFileInfo.ts new file mode 100644 index 000000000..ccdbfa721 --- /dev/null +++ b/client/src/pages/api/plugins/kb/file/getFileInfo.ts @@ -0,0 +1,43 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@/service/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authUser } from '@/service/utils/auth'; +import { GridFSStorage } from '@/service/lib/gridfs'; +import { OtherFileId } from '@/constants/kb'; +import type { FileInfo } from '@/types/plugin'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + + const { fileId } = req.query as { kbId: string; fileId: string }; + // 凭证校验 + const { userId } = await authUser({ req, authToken: true }); + + if (fileId === OtherFileId) { + return jsonRes(res, { + data: { + id: OtherFileId, + size: 0, + filename: 'kb.Other Data', + uploadDate: new Date(), + encoding: '', + contentType: '' + } + }); + } + + const gridFs = new GridFSStorage('dataset', userId); + + const file = await gridFs.findAndAuthFile(fileId); + + jsonRes(res, { + data: file + }); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/client/src/pages/api/plugins/kb/file/list.ts b/client/src/pages/api/plugins/kb/file/list.ts new file mode 100644 index 000000000..b757cb8be --- /dev/null +++ b/client/src/pages/api/plugins/kb/file/list.ts @@ -0,0 +1,82 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@/service/response'; +import { connectToDatabase, TrainingData } from '@/service/mongo'; +import { authUser } from '@/service/utils/auth'; +import { GridFSStorage } from '@/service/lib/gridfs'; +import { PgClient } from '@/service/pg'; +import { PgTrainingTableName } from '@/constants/plugin'; +import { KbFileItemType } from '@/types/plugin'; +import { FileStatusEnum, OtherFileId } from '@/constants/kb'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + + const { kbId } = req.query as { kbId: string }; + // 凭证校验 + const { userId } = await authUser({ req, authToken: true }); + + const gridFs = new GridFSStorage('dataset', userId); + const bucket = gridFs.GridFSBucket(); + + const files = await bucket + .find({ ['metadata.kbId']: kbId }) + .sort({ _id: -1 }) + .toArray(); + + async function GetOtherData() { + return { + id: OtherFileId, + size: 0, + filename: 'kb.Other Data', + uploadTime: new Date(), + status: (await TrainingData.findOne({ userId, kbId, file_id: '' })) + ? FileStatusEnum.embedding + : FileStatusEnum.ready, + chunkLength: await PgClient.count(PgTrainingTableName, { + fields: ['kb_id'], + where: [ + ['user_id', userId], + 'AND', + ['kb_id', kbId], + "AND (file_id IS NULL OR file_id = '')" + ] + }) + }; + } + + const data = await Promise.all([ + GetOtherData(), + ...files.map(async (file) => { + return { + id: String(file._id), + size: file.length, + filename: file.filename, + uploadTime: file.uploadDate, + status: (await TrainingData.findOne({ userId, kbId, file_id: file._id })) + ? FileStatusEnum.embedding + : FileStatusEnum.ready, + chunkLength: await PgClient.count(PgTrainingTableName, { + fields: ['kb_id'], + where: [ + ['user_id', userId], + 'AND', + ['kb_id', kbId], + 'AND', + ['file_id', String(file._id)] + ] + }) + }; + }) + ]); + + jsonRes(res, { + data: data.flat().filter((item) => item.chunkLength > 0) + }); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/client/src/pages/kb/detail/components/DataCard.tsx b/client/src/pages/kb/detail/components/DataCard.tsx index d3b6033da..a22b9fbdf 100644 --- a/client/src/pages/kb/detail/components/DataCard.tsx +++ b/client/src/pages/kb/detail/components/DataCard.tsx @@ -1,12 +1,13 @@ -import React, { useCallback, useState, useRef } from 'react'; -import { Box, Card, IconButton, Flex, Button, Input, Grid } from '@chakra-ui/react'; +import React, { useCallback, useState, useRef, useMemo } from 'react'; +import { Box, Card, IconButton, Flex, Button, Grid, Image } from '@chakra-ui/react'; import type { KbDataItemType } from '@/types/plugin'; import { usePagination } from '@/hooks/usePagination'; import { getKbDataList, getExportDataList, delOneKbDataByDataId, - getTrainingData + getTrainingData, + getFileInfoById } from '@/api/plugins/kb'; import { DeleteIcon, RepeatIcon } from '@chakra-ui/icons'; import { fileDownload } from '@/utils/file'; @@ -18,12 +19,18 @@ import { debounce } from 'lodash'; import { getErrText } from '@/utils/tools'; import { useConfirm } from '@/hooks/useConfirm'; import { useTranslation } from 'react-i18next'; +import { useRouter } from 'next/router'; import MyIcon from '@/components/Icon'; import MyTooltip from '@/components/MyTooltip'; +import MyInput from '@/components/MyInput'; +import { fileImgs } from '@/constants/common'; +import { useRequest } from '@/hooks/useRequest'; const DataCard = ({ kbId }: { kbId: string }) => { const BoxRef = useRef(null); const lastSearch = useRef(''); + const router = useRouter(); + const { fileId = '' } = router.query as { fileId: string }; const { t } = useTranslation(); const [searchText, setSearchText] = useState(''); const { toast } = useToast(); @@ -45,7 +52,8 @@ const DataCard = ({ kbId }: { kbId: string }) => { pageSize: 24, params: { kbId, - searchText + searchText, + fileId }, onChange() { if (BoxRef.current) { @@ -73,7 +81,7 @@ const DataCard = ({ kbId }: { kbId: string }) => { ); // get al data and export csv - const { mutate: onclickExport, isLoading: isLoadingExport = false } = useMutation({ + const { mutate: onclickExport, isLoading: isLoadingExport = false } = useRequest({ mutationFn: () => getExportDataList(kbId), onSuccess(res) { const text = Papa.unparse({ @@ -85,20 +93,12 @@ const DataCard = ({ kbId }: { kbId: string }) => { type: 'text/csv', filename: 'data.csv' }); - toast({ - title: '导出成功，下次导出需要半小时后', - status: 'success' - }); }, - onError(err: any) { - toast({ - title: getErrText(err, '导出异常'), - status: 'error' - }); - console.log(err); - } + successToast: '导出成功，下次导出需要半小时后', + errorToast: '导出异常' }); + // get first page data const getFirstData = useCallback( debounce(() => { getData(1); @@ -113,57 +113,78 @@ const DataCard = ({ kbId }: { kbId: string }) => { enabled: qaListLen > 0 || vectorListLen > 0 }); + // get file info + const { data: fileInfo } = useQuery(['getFileInfo', fileId], () => getFileInfoById(fileId)); + const fileIcon = useMemo( + () => + fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(fileInfo?.filename || ''))?.src, + [fileInfo?.filename] + ); + return ( - - - 知识库数据: {total}组 - + + + {''} + {t(fileInfo?.filename || 'Filename')} + + } + size={['sm', 'md']} aria-label={'refresh'} variant={'base'} isLoading={isLoading} - mr={[2, 4]} - size={'sm'} onClick={() => { getData(pageNum); getTrainingData({ kbId, init: true }); }} /> - - - {qaListLen > 0 || vectorListLen > 0 ? ( - - {qaListLen > 0 ? `${qaListLen}条数据正在拆分，` : ''} - {vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引，` : ''} - 请耐心等待... + + + + {total}组 - ) : ( - 所有数据已就绪~ - )} + + {(qaListLen > 0 || vectorListLen > 0) && ( + <> + ({qaListLen > 0 ? `${qaListLen}条数据正在拆分，` : ''} + {vectorListLen > 0 ? `${vectorListLen}条数据正在生成索引，` : ''} + 请耐心等待... ) + + )} + + - + } + w={['200px', '300px']} placeholder="根据匹配知识，预期答案和来源进行搜索" + value={searchText} onChange={(e) => { setSearchText(e.target.value); getFirstData(); diff --git a/client/src/pages/kb/detail/components/FileCard.tsx b/client/src/pages/kb/detail/components/FileCard.tsx new file mode 100644 index 000000000..4f03c4ea7 --- /dev/null +++ b/client/src/pages/kb/detail/components/FileCard.tsx @@ -0,0 +1,196 @@ +import React, { useCallback, useState, useRef, useMemo } from 'react'; +import { + Box, + Flex, + TableContainer, + Table, + Thead, + Tr, + Th, + Td, + Tbody, + Image +} from '@chakra-ui/react'; +import { getKbFiles, deleteKbFileById } from '@/api/plugins/kb'; +import { useQuery } from '@tanstack/react-query'; +import { useToast } from '@/hooks/useToast'; +import { debounce } from 'lodash'; +import { formatFileSize } from '@/utils/tools'; +import { useConfirm } from '@/hooks/useConfirm'; +import { useTranslation } from 'react-i18next'; +import MyIcon from '@/components/Icon'; +import MyInput from '@/components/MyInput'; +import dayjs from 'dayjs'; +import { fileImgs } from '@/constants/common'; +import { useRequest } from '@/hooks/useRequest'; +import { useLoading } from '@/hooks/useLoading'; +import { FileStatusEnum } from '@/constants/kb'; +import { useRouter } from 'next/router'; + +const FileCard = ({ kbId }: { kbId: string }) => { + const BoxRef = useRef(null); + const lastSearch = useRef(''); + const router = useRouter(); + const { t } = useTranslation(); + const [searchText, setSearchText] = useState(''); + const { Loading } = useLoading(); + const { openConfirm, ConfirmModal } = useConfirm({ + content: t('kb.Confirm to delete the file') + }); + + const { + data: files = [], + refetch, + isInitialLoading + } = useQuery(['getFiles', kbId], () => getKbFiles(kbId), { + refetchInterval: 6000, + refetchOnWindowFocus: true + }); + const formatFiles = useMemo( + () => + files.map((file) => ({ + ...file, + icon: fileImgs.find((item) => new RegExp(item.suffix, 'gi').test(file.filename))?.src + })), + [files] + ); + const totalDataLength = useMemo( + () => files.reduce((sum, item) => sum + item.chunkLength, 0), + [files] + ); + + const { mutate: onDeleteFile, isLoading } = useRequest({ + mutationFn: (fileId: string) => + deleteKbFileById({ + fileId, + kbId + }), + onSuccess() { + refetch(); + }, + successToast: t('common.Delete Success'), + errorToast: t('common.Delete Failed') + }); + + const statusMap = { + [FileStatusEnum.embedding]: { + color: 'myGray.500', + text: t('file.Embedding') + }, + [FileStatusEnum.ready]: { + color: 'green.500', + text: t('file.Ready') + } + }; + + return ( + + + + {t('kb.Files', { total: files.length })} + + + + } + w={['100%', '200px']} + placeholder={t('common.Search') || ''} + value={searchText} + onChange={(e) => { + setSearchText(e.target.value); + }} + onBlur={() => { + if (searchText === lastSearch.current) return; + refetch(); + }} + onKeyDown={(e) => { + if (searchText === lastSearch.current) return; + if (e.key === 'Enter') { + refetch(); + } + }} + /> + + + + + + + + + + + + + + + {formatFiles.map((file) => ( + + router.push({ + query: { + kbId, + fileId: file.id, + currentTab: 'dataCard' + } + }) + } + > + + + + + + + + ))} + +
{t('kb.Filename')} + {t('kb.Chunk Length')}({totalDataLength}) + {t('kb.Upload Time')}{t('kb.File Size')}{t('common.Status')} +
+ {''} + + {t(file.filename)} + + + {file.chunkLength} + {dayjs(file.uploadTime).format('YYYY/MM/DD HH:mm')}{formatFileSize(file.size)} + {statusMap[file.status].text} + e.stopPropagation()}> + + openConfirm(() => { + onDeleteFile(file.id); + })() + } + /> +
+ + + + + + ); +}; + +export default React.memo(FileCard); diff --git a/client/src/pages/kb/detail/components/Import/Chunk.tsx b/client/src/pages/kb/detail/components/Import/Chunk.tsx index d28516795..759125f4b 100644 --- a/client/src/pages/kb/detail/components/Import/Chunk.tsx +++ b/client/src/pages/kb/detail/components/Import/Chunk.tsx @@ -86,7 +86,7 @@ const ChunkImport = ({ kbId }: { kbId: string }) => { router.replace({ query: { kbId, - currentTab: 'data' + currentTab: 'dataset' } }); }, diff --git a/client/src/pages/kb/detail/components/Import/Csv.tsx b/client/src/pages/kb/detail/components/Import/Csv.tsx index 15d701631..0d134ca40 100644 --- a/client/src/pages/kb/detail/components/Import/Csv.tsx +++ b/client/src/pages/kb/detail/components/Import/Csv.tsx @@ -73,7 +73,7 @@ const CsvImport = ({ kbId }: { kbId: string }) => { router.replace({ query: { kbId, - currentTab: 'data' + currentTab: 'dataset' } }); }, diff --git a/client/src/pages/kb/detail/components/Import/QA.tsx b/client/src/pages/kb/detail/components/Import/QA.tsx index 5b3f875a9..e69c003d1 100644 --- a/client/src/pages/kb/detail/components/Import/QA.tsx +++ b/client/src/pages/kb/detail/components/Import/QA.tsx @@ -74,7 +74,7 @@ const QAImport = ({ kbId }: { kbId: string }) => { router.replace({ query: { kbId, - currentTab: 'data' + currentTab: 'dataset' } }); }, diff --git a/client/src/pages/kb/detail/components/InputDataModal.tsx b/client/src/pages/kb/detail/components/InputDataModal.tsx index c45f87975..eb63ee803 100644 --- a/client/src/pages/kb/detail/components/InputDataModal.tsx +++ b/client/src/pages/kb/detail/components/InputDataModal.tsx @@ -261,6 +261,7 @@ export function RawFileText({ fileId, filename = '', ...props }: RawFileTextProp import('./components/DataCard'), { + ssr: false +}); const ImportData = dynamic(() => import('./components/Import'), { ssr: false }); @@ -33,7 +36,8 @@ const Test = dynamic(() => import('./components/Test'), { }); enum TabEnum { - data = 'data', + dataCard = 'dataCard', + dataset = 'dataset', import = 'import', test = 'test', info = 'info' @@ -49,7 +53,7 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` } const { kbDetail, getKbDetail } = useUserStore(); const tabList = useRef([ - { label: '数据集', id: TabEnum.data, icon: 'overviewLight' }, + { label: '数据集', id: TabEnum.dataset, icon: 'overviewLight' }, { label: '导入数据', id: TabEnum.import, icon: 'importLight' }, { label: '搜索测试', id: TabEnum.test, icon: 'kbTest' }, { label: '配置', id: TabEnum.info, icon: 'settingLight' } @@ -86,9 +90,17 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` } }); const { data: trainingQueueLen = 0 } = useQuery(['getTrainingQueueLen'], getTrainingQueueLen, { - refetchInterval: 5000 + refetchInterval: 10000 }); + useEffect(() => { + return () => { + try { + delEmptyFiles(kbId); + } catch (error) {} + }; + }, [kbId]); + return ( <> @@ -141,7 +153,7 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` } px={3} borderRadius={'md'} _hover={{ bg: 'myGray.100' }} - onClick={() => router.back()} + onClick={() => router.replace('/kb/list')} > - {currentTab === TabEnum.data && } + {currentTab === TabEnum.dataset && } + {currentTab === TabEnum.dataCard && } {currentTab === TabEnum.import && } {currentTab === TabEnum.test && } {currentTab === TabEnum.info && } @@ -187,7 +200,7 @@ const Detail = ({ kbId, currentTab }: { kbId: string; currentTab: `${TabEnum}` } }; export async function getServerSideProps(context: any) { - const currentTab = context?.query?.currentTab || TabEnum.data; + const currentTab = context?.query?.currentTab || TabEnum.dataset; const kbId = context?.query?.kbId; return { diff --git a/client/src/service/events/generateQA.ts b/client/src/service/events/generateQA.ts index efdb43e1e..ef1ebf3bd 100644 --- a/client/src/service/events/generateQA.ts +++ b/client/src/service/events/generateQA.ts @@ -158,7 +158,7 @@ A2: console.log('openai error: 生成QA错误'); console.log(err.response?.status, err.response?.statusText, err.response?.data); } else { - console.log('生成QA错误:', err); + addLog.error('生成 QA 错误', err); } // message error or openai account error diff --git a/client/src/service/events/generateVector.ts b/client/src/service/events/generateVector.ts index 3cc486a14..8d17f6cef 100644 --- a/client/src/service/events/generateVector.ts +++ b/client/src/service/events/generateVector.ts @@ -96,9 +96,7 @@ export async function generateVector(): Promise { data: err.response?.data }); } else { - addLog.info('openai error: 生成向量错误', { - err - }); + addLog.error('openai error: 生成向量错误', err); } // message error or openai account error diff --git a/client/src/service/lib/gridfs.ts b/client/src/service/lib/gridfs.ts index 3804091f7..76ed806ed 100644 --- a/client/src/service/lib/gridfs.ts +++ b/client/src/service/lib/gridfs.ts @@ -2,19 +2,12 @@ import mongoose, { Types } from 'mongoose'; import fs from 'fs'; import fsp from 'fs/promises'; import { ERROR_ENUM } from '../errorCode'; +import type { FileInfo } from '@/types/plugin'; enum BucketNameEnum { dataset = 'dataset' } -type FileInfo = { - id: string; - filename: string; - size: number; - contentType: string; - encoding: string; -}; - export class GridFSStorage { readonly type = 'gridfs'; readonly bucket: `${BucketNameEnum}`; @@ -88,6 +81,7 @@ export class GridFSStorage { filename: file.filename, contentType: file.metadata?.contentType, encoding: file.metadata?.encoding, + uploadDate: file.uploadDate, size: file.length }; } diff --git a/client/src/service/utils/auth.ts b/client/src/service/utils/auth.ts index de0068515..8c08486d4 100644 --- a/client/src/service/utils/auth.ts +++ b/client/src/service/utils/auth.ts @@ -155,7 +155,7 @@ export const authUser = async ({ })(); return { - userId: uid, + userId: String(uid), appId, authType, user diff --git a/client/src/types/plugin.d.ts b/client/src/types/plugin.d.ts index 585eb729c..080f551b2 100644 --- a/client/src/types/plugin.d.ts +++ b/client/src/types/plugin.d.ts @@ -1,3 +1,4 @@ +import { FileStatusEnum } from '@/constants/kb'; import { VectorModelItemType } from './model'; import type { kbSchema } from './mongoSchema'; @@ -22,6 +23,15 @@ export interface KbItemType { tags: string; } +export type KbFileItemType = { + id: string; + size: number; + filename: string; + uploadTime: Date; + chunkLength: number; + status: `${FileStatusEnum}`; +}; + export type DatasetItemType = { q: string; // 提问词 a: string; // 原文 @@ -44,3 +54,12 @@ export type FetchResultItem = { url: string; content: string; }; + +export type FileInfo = { + id: string; + filename: string; + size: number; + contentType: string; + encoding: string; + uploadDate: Date; +}; diff --git a/client/src/utils/tools.ts b/client/src/utils/tools.ts index 618260957..a6420edcb 100644 --- a/client/src/utils/tools.ts +++ b/client/src/utils/tools.ts @@ -59,6 +59,9 @@ export const Obj2Query = (obj: Record) => { return queryParams.toString(); }; +/** + * parse string to query object + */ export const parseQueryString = (str: string) => { const queryObject: Record = {}; @@ -125,6 +128,16 @@ export const formatTimeToChatTime = (time: Date) => { return target.format('YYYY/M/D'); }; +export const formatFileSize = (bytes: number): string => { + if (bytes === 0) return '0 B'; + + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; +}; + export const hasVoiceApi = typeof window !== 'undefined' && 'speechSynthesis' in window; /** * voice broadcast