Training status (#4424)

* dataset data training state (#4311) * dataset data training state * fix * fix ts * fix * fix api format * fix * fix * perf: count training * format * fix: dataset training state (#4417) * fix * add test * fix * fix * fix test * fix test * perf: training count * count * loading status --------- Co-authored-by: heheer <heheer@sealos.io>
2025-04-02 10:53:15 +08:00
parent 5839325f77
commit 27332743c7
33 changed files with 1383 additions and 19 deletions
--- a/packages/global/core/dataset/type.d.ts
+++ b/packages/global/core/dataset/type.d.ts
@@ -163,6 +163,7 @@ export type DatasetTrainingSchemaType = {
  weight: number;
  indexes: Omit<DatasetDataIndexItemType, 'dataId'>[];
  retryCount: number;
+  errorMsg?: string;
 };

 export type CollectionWithDatasetType = DatasetCollectionSchemaType & {
@@ -216,6 +217,7 @@ export type DatasetCollectionItemType = CollectionWithDatasetType & {
  file?: DatasetFileSchema;
  permission: DatasetPermission;
  indexAmount: number;
+  errorCount?: number;
 };

 /* ================= data ===================== */
--- a/packages/service/core/dataset/training/schema.ts
+++ b/packages/service/core/dataset/training/schema.ts
@@ -98,7 +98,9 @@ const TrainingDataSchema = new Schema({
      }
    ],
    default: []
-  }
+  },
+
+  errorMsg: String
 });

 TrainingDataSchema.virtual('dataset', {
--- a/packages/web/components/common/Icon/constants.ts
+++ b/packages/web/components/common/Icon/constants.ts
@@ -67,6 +67,7 @@ export const iconPaths = {
  'common/list': () => import('./icons/common/list.svg'),
  'common/loading': () => import('./icons/common/loading.svg'),
  'common/logLight': () => import('./icons/common/logLight.svg'),
+  'common/maximize': () => import('./icons/common/maximize.svg'),
  'common/microsoft': () => import('./icons/common/microsoft.svg'),
  'common/model': () => import('./icons/common/model.svg'),
  'common/monitor': () => import('./icons/common/monitor.svg'),
@@ -85,6 +86,7 @@ export const iconPaths = {
  'common/rightArrowFill': () => import('./icons/common/rightArrowFill.svg'),
  'common/rightArrowLight': () => import('./icons/common/rightArrowLight.svg'),
  'common/routePushLight': () => import('./icons/common/routePushLight.svg'),
+  'common/running': () => import('./icons/common/running.svg'),
  'common/saveFill': () => import('./icons/common/saveFill.svg'),
  'common/searchLight': () => import('./icons/common/searchLight.svg'),
  'common/select': () => import('./icons/common/select.svg'),
--- a/packages/web/components/common/Icon/icons/common/check.svg
+++ b/packages/web/components/common/Icon/icons/common/check.svg
@@ -1,3 +1,3 @@
-<svg xmlns="http://www.w3.org/2000/svg"  viewBox="0 0 11 10" fill="none">
-  <path fill-rule="evenodd" clip-rule="evenodd" d="M8.82531 2.05806C9.06939 1.81398 9.46512 1.81398 9.70919 2.05806C9.95327 2.30214 9.95327 2.69786 9.7092 2.94194L5.12586 7.52528C4.88178 7.76935 4.48606 7.76935 4.24198 7.52528L2.15864 5.44194C1.91457 5.19786 1.91457 4.80214 2.15864 4.55806C2.40272 4.31398 2.79845 4.31398 3.04253 4.55806L4.68392 6.19945L8.82531 2.05806Z" />
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 12 12" fill="none">
+  <path fill-rule="evenodd" clip-rule="evenodd" d="M9.46964 2.46967C9.76253 2.17678 10.2374 2.17678 10.5303 2.46967C10.8232 2.76256 10.8232 3.23744 10.5303 3.53033L5.0303 9.03033C4.73741 9.32322 4.26253 9.32322 3.96964 9.03033L1.46964 6.53033C1.17675 6.23744 1.17675 5.76256 1.46964 5.46967C1.76253 5.17678 2.23741 5.17678 2.5303 5.46967L4.49997 7.43934L9.46964 2.46967Z" />
 </svg>
--- a/packages/web/components/common/Icon/icons/common/maximize.svg
+++ b/packages/web/components/common/Icon/icons/common/maximize.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 10 11" fill="none">
+  <path fill-rule="evenodd" clip-rule="evenodd" d="M6.24998 2.22347C6.01986 2.22347 5.83331 2.03692 5.83331 1.8068C5.83331 1.57668 6.01986 1.39014 6.24998 1.39014H8.74998C8.9801 1.39014 9.16665 1.57668 9.16665 1.8068V4.3068C9.16665 4.53692 8.9801 4.72347 8.74998 4.72347C8.51986 4.72347 8.33331 4.53692 8.33331 4.3068V2.81273L6.12794 5.0181C5.96522 5.18082 5.7014 5.18082 5.53869 5.0181C5.37597 4.85538 5.37597 4.59156 5.53869 4.42884L7.74406 2.22347H6.24998ZM4.46127 6.09551C4.62399 6.25823 4.62399 6.52205 4.46127 6.68476L2.2559 8.89014H3.74998C3.9801 8.89014 4.16665 9.07669 4.16665 9.3068C4.16665 9.53692 3.9801 9.72347 3.74998 9.72347H1.24998C1.13947 9.72347 1.03349 9.67957 0.955352 9.60143C0.877212 9.52329 0.833313 9.41731 0.833313 9.3068L0.833313 6.8068C0.833313 6.57668 1.01986 6.39014 1.24998 6.39014C1.4801 6.39014 1.66665 6.57668 1.66665 6.8068L1.66665 8.30088L3.87202 6.09551C4.03474 5.93279 4.29856 5.93279 4.46127 6.09551Z" />
+</svg>
--- a/packages/web/components/common/Icon/icons/common/running.svg
+++ b/packages/web/components/common/Icon/icons/common/running.svg
@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20" fill="none">
+  <path fill-rule="evenodd" clip-rule="evenodd" d="M3.42335 9.70025C3.42335 6.06906 6.36701 3.1254 9.9982 3.1254C11.8153 3.1254 13.6728 3.9104 14.9391 5.32281C16.1863 6.71386 16.8964 8.75372 16.3686 11.3689C16.2731 11.8419 16.5792 12.3028 17.0522 12.3982C17.5252 12.4937 17.9861 12.1876 18.0815 11.7146C18.7147 8.57736 17.8683 5.97217 16.2402 4.15628C14.6313 2.36174 12.2949 1.37793 9.9982 1.37793C5.40191 1.37793 1.67588 5.10396 1.67588 9.70025C1.67588 11.6523 2.5328 14.2506 4.35082 15.8991L3.75846 15.8976C3.27592 15.8964 2.88375 16.2866 2.88253 16.7692C2.88132 17.2517 3.27152 17.6439 3.75407 17.6451L6.84117 17.6529C7.17714 17.6537 7.4693 17.4648 7.6162 17.1871C7.69971 17.0534 7.74807 16.8955 7.74833 16.7262L7.75314 13.6037C7.75388 13.1211 7.3633 12.7293 6.88075 12.7286C6.3982 12.7278 6.00641 13.1184 6.00567 13.601L6.00354 14.9858C4.32556 13.8215 3.42335 11.4678 3.42335 9.70025ZM16.8177 14.5519C16.8177 15.1962 16.2954 15.7185 15.651 15.7185C15.0067 15.7185 14.4844 15.1962 14.4844 14.5519C14.4844 13.9075 15.0067 13.3852 15.651 13.3852C16.2954 13.3852 16.8177 13.9075 16.8177 14.5519ZM12.6957 17.7912C13.3401 17.7912 13.8624 17.2689 13.8624 16.6246C13.8624 15.9802 13.3401 15.4579 12.6957 15.4579C12.0514 15.4579 11.5291 15.9802 11.5291 16.6246C11.5291 17.2689 12.0514 17.7912 12.6957 17.7912Z" fill="#3370FF"/>
+</svg>
--- a/packages/web/hooks/useScrollPagination.tsx
+++ b/packages/web/hooks/useScrollPagination.tsx
@@ -308,7 +308,13 @@ export function useScrollPagination<
      );

      return (
-        <MyBox ref={ref} h={'100%'} overflow={'auto'} isLoading={isLoading} {...props}>
+        <MyBox
+          ref={ref}
+          h={'100%'}
+          overflow={'auto'}
+          isLoading={isLoading || isLoadingProp}
+          {...props}
+        >
          {scrollLoadType === 'top' && total > 0 && isLoading && (
            <Box mt={2} fontSize={'xs'} color={'blackAlpha.500'} textAlign={'center'}>
              {t('common:common.is_requesting')}
--- a/packages/web/i18n/en/common.json
+++ b/packages/web/i18n/en/common.json
@@ -1,5 +1,6 @@
 {
  "App": "Application",
+  "Click_to_expand": "Click to expand",
  "Download": "Download",
  "Export": "Export",
  "FAQ.ai_point_a": "Each time you use the AI model, a certain amount of AI points will be deducted. For detailed calculation standards, please refer to the 'AI Points Calculation Standards' above.\nToken calculation uses the same formula as GPT-3.5, where 1 Token ≈ 0.7 Chinese characters ≈ 0.9 English words. Consecutive characters may be considered as 1 Token.",
@@ -538,6 +539,7 @@
  "core.dataset.collection.metadata.source name": "Source Name",
  "core.dataset.collection.metadata.source size": "Source Size",
  "core.dataset.collection.status.active": "Ready",
+  "core.dataset.collection.status.error": "Error",
  "core.dataset.collection.sync.result.sameRaw": "Content Unchanged, No Update Needed",
  "core.dataset.collection.sync.result.success": "Sync Started",
  "core.dataset.data.Data Content": "Related Data Content",
--- a/packages/web/i18n/en/dataset.json
+++ b/packages/web/i18n/en/dataset.json
@@ -28,9 +28,24 @@
  "custom_data_process_params_desc": "Customize data processing rules",
  "custom_split_sign_tip": "Allows you to chunk according to custom delimiters. \nUsually used for processed data, using specific separators for precise chunking. \nYou can use the | symbol to represent multiple splitters, such as: \".|.\" to represent a period in Chinese and English.\n\nTry to avoid using special symbols related to regular, such as: * () [] {}, etc.",
  "data_amount": "{{dataAmount}} Datas, {{indexAmount}} Indexes",
+  "data_error_amount": "{{errorAmount}} Group training exception",
  "data_index_num": "Index {{index}}",
  "data_process_params": "Params",
  "data_process_setting": "Processing config",
+  "dataset.Chunk_Number": "Block number",
+  "dataset.Completed": "Finish",
+  "dataset.Delete_Chunk": "delete",
+  "dataset.Edit_Chunk": "edit",
+  "dataset.Error_Message": "Report an error message",
+  "dataset.No_Error": "No exception information yet",
+  "dataset.Operation": "operate",
+  "dataset.ReTrain": "Retrain",
+  "dataset.Training Process": "Training status",
+  "dataset.Training_Count": "{{count}} Group training",
+  "dataset.Training_Errors": "Errors",
+  "dataset.Training_QA": "{{count}} Group Q&A pair training",
+  "dataset.Training_Status": "Training status",
+  "dataset.Training_Waiting": "Need to wait for {{count}} group data",
  "dataset.Unsupported operation": "dataset.Unsupported operation",
  "dataset.no_collections": "No datasets available",
  "dataset.no_tags": "No tags available",
@@ -82,6 +97,13 @@
  "preview_chunk_empty": "Unable to read the contents of the file",
  "preview_chunk_intro": "A total of {{total}} blocks, up to 10",
  "preview_chunk_not_selected": "Click on the file on the left to preview",
+  "process.Auto_Index": "Automatic index generation",
+  "process.Get QA": "Q&A extraction",
+  "process.Image_Index": "Image index generation",
+  "process.Is_Ready": "Ready",
+  "process.Parsing": "Parsing",
+  "process.Vectorizing": "Index vectorization",
+  "process.Waiting": "Queue",
  "rebuild_embedding_start_tip": "Index model switching task has started",
  "rebuilding_index_count": "Number of indexes being rebuilt: {{count}}",
  "request_headers": "Request headers, will automatically append 'Bearer '",
@@ -114,7 +136,10 @@
  "tag.total_tags": "Total {{total}} tags",
  "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "The Dataset has indexes that are being trained or rebuilt",
  "total_num_files": "Total {{total}} files",
+  "training.Error": "{{count}} Group exception",
+  "training.Normal": "Normal",
  "training_mode": "Chunk mode",
+  "training_ready": "{{count}} Group",
  "vector_model_max_tokens_tip": "Each chunk of data has a maximum length of 3000 tokens",
  "vllm_model": "Image understanding model",
  "website_dataset": "Website Sync",
--- a/packages/web/i18n/zh-CN/common.json
+++ b/packages/web/i18n/zh-CN/common.json
@@ -1,5 +1,6 @@
 {
  "App": "应用",
+  "Click_to_expand": "点击查看详情",
  "Download": "下载",
  "Export": "导出",
  "FAQ.ai_point_a": "每次调用AI模型时，都会消耗一定的AI积分。具体的计算标准可参考上方的“AI 积分计算标准”。\nToken计算采用GPT3.5相同公式，1Token≈0.7中文字符≈0.9英文单词，连续出现的字符可能被认为是1个Tokens。",
@@ -541,6 +542,7 @@
  "core.dataset.collection.metadata.source name": "来源名",
  "core.dataset.collection.metadata.source size": "来源大小",
  "core.dataset.collection.status.active": "已就绪",
+  "core.dataset.collection.status.error": "训练异常",
  "core.dataset.collection.sync.result.sameRaw": "内容未变动，无需更新",
  "core.dataset.collection.sync.result.success": "开始同步",
  "core.dataset.data.Data Content": "相关数据内容",
--- a/packages/web/i18n/zh-CN/dataset.json
+++ b/packages/web/i18n/zh-CN/dataset.json
@@ -28,9 +28,24 @@
  "custom_data_process_params_desc": "自定义设置数据处理规则",
  "custom_split_sign_tip": "允许你根据自定义的分隔符进行分块。通常用于已处理好的数据，使用特定的分隔符来精确分块。可以使用 | 符号表示多个分割符，例如：“。|.” 表示中英文句号。\n尽量避免使用正则相关特殊符号，例如: * () [] {} 等。",
  "data_amount": "{{dataAmount}} 组数据, {{indexAmount}} 组索引",
+  "data_error_amount": "{{errorAmount}} 组训练异常",
  "data_index_num": "索引 {{index}}",
  "data_process_params": "处理参数",
  "data_process_setting": "数据处理配置",
+  "dataset.Chunk_Number": "分块号",
+  "dataset.Completed": "完成",
+  "dataset.Delete_Chunk": "删除",
+  "dataset.Edit_Chunk": "编辑",
+  "dataset.Error_Message": "报错信息",
+  "dataset.No_Error": "暂无异常信息",
+  "dataset.Operation": "操作",
+  "dataset.ReTrain": "重试",
+  "dataset.Training Process": "训练状态",
+  "dataset.Training_Count": "{{count}} 组训练中",
+  "dataset.Training_Errors": "异常 ({{count}})",
+  "dataset.Training_QA": "{{count}} 组问答对训练中",
+  "dataset.Training_Status": "训练状态",
+  "dataset.Training_Waiting": "需等待 {{count}} 组数据",
  "dataset.Unsupported operation": "操作不支持",
  "dataset.no_collections": "暂无数据集",
  "dataset.no_tags": "暂无标签",
@@ -82,6 +97,14 @@
  "preview_chunk_empty": "无法读取该文件内容",
  "preview_chunk_intro": "共 {{total}} 个分块，最多展示 10 个",
  "preview_chunk_not_selected": "点击左侧文件后进行预览",
+  "process.Auto_Index": "自动索引生成",
+  "process.Get QA": "问答对提取",
+  "process.Image_Index": "图片索引生成",
+  "process.Is_Ready": "已就绪",
+  "process.Is_Ready_Count": "{{count}} 组已就绪",
+  "process.Parsing": "内容解析中",
+  "process.Vectorizing": "索引向量化",
+  "process.Waiting": "排队中",
  "rebuild_embedding_start_tip": "切换索引模型任务已开始",
  "rebuilding_index_count": "重建中索引数量：{{count}}",
  "request_headers": "请求头参数，会自动补充 Bearer",
@@ -114,7 +137,10 @@
  "tag.total_tags": "共{{total}}个标签",
  "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "知识库有训练中或正在重建的索引",
  "total_num_files": "共 {{total}} 个文件",
+  "training.Error": "{{count}} 组异常",
+  "training.Normal": "正常",
  "training_mode": "处理方式",
+  "training_ready": "{{count}} 组",
  "vector_model_max_tokens_tip": "每个分块数据，最大长度为 3000 tokens",
  "vllm_model": "图片理解模型",
  "website_dataset": "Web 站点同步",
--- a/packages/web/i18n/zh-Hant/common.json
+++ b/packages/web/i18n/zh-Hant/common.json
@@ -1,5 +1,6 @@
 {
  "App": "應用程式",
+  "Click_to_expand": "點擊查看詳情",
  "Download": "下載",
  "Export": "匯出",
  "FAQ.ai_point_a": "每次呼叫 AI 模型時，都會消耗一定數量的 AI 點數。詳細的計算標準請參考上方的「AI 點數計算標準」。\nToken 計算採用與 GPT3.5 相同的公式，1 Token ≈ 0.7 個中文字 ≈ 0.9 個英文單字，連續出現的字元可能會被視為 1 個 Token。",
@@ -537,6 +538,7 @@
  "core.dataset.collection.metadata.source name": "來源名稱",
  "core.dataset.collection.metadata.source size": "來源大小",
  "core.dataset.collection.status.active": "已就緒",
+  "core.dataset.collection.status.error": "訓練異常",
  "core.dataset.collection.sync.result.sameRaw": "內容未變更，無需更新",
  "core.dataset.collection.sync.result.success": "開始同步",
  "core.dataset.data.Data Content": "相關資料內容",
--- a/packages/web/i18n/zh-Hant/dataset.json
+++ b/packages/web/i18n/zh-Hant/dataset.json
@@ -28,9 +28,24 @@
  "custom_data_process_params_desc": "自訂資料處理規則",
  "custom_split_sign_tip": "允許你根據自定義的分隔符進行分塊。\n通常用於已處理好的數據，使用特定的分隔符來精確分塊。\n可以使用 | 符號表示多個分割符，例如：“。|.” 表示中英文句號。\n\n盡量避免使用正則相關特殊符號，例如: * () [] {} 等。",
  "data_amount": "{{dataAmount}} 組數據, {{indexAmount}} 組索引",
+  "data_error_amount": "{{errorAmount}} 組訓練異常",
  "data_index_num": "索引 {{index}}",
  "data_process_params": "處理參數",
  "data_process_setting": "資料處理設定",
+  "dataset.Chunk_Number": "分塊號",
+  "dataset.Completed": "完成",
+  "dataset.Delete_Chunk": "刪除",
+  "dataset.Edit_Chunk": "編輯",
+  "dataset.Error_Message": "報錯信息",
+  "dataset.No_Error": "暫無異常信息",
+  "dataset.Operation": "操作",
+  "dataset.ReTrain": "重試",
+  "dataset.Training Process": "訓練狀態",
+  "dataset.Training_Count": "{{count}} 組訓練中",
+  "dataset.Training_Errors": "異常",
+  "dataset.Training_QA": "{{count}} 組問答對訓練中",
+  "dataset.Training_Status": "訓練狀態",
+  "dataset.Training_Waiting": "需等待 {{count}} 組數據",
  "dataset.Unsupported operation": "操作不支持",
  "dataset.no_collections": "尚無資料集",
  "dataset.no_tags": "尚無標籤",
@@ -82,6 +97,13 @@
  "preview_chunk_empty": "無法讀取該文件內容",
  "preview_chunk_intro": "共 {{total}} 個分塊，最多展示 10 個",
  "preview_chunk_not_selected": "點擊左側文件後進行預覽",
+  "process.Auto_Index": "自動索引生成",
+  "process.Get QA": "問答對提取",
+  "process.Image_Index": "圖片索引生成",
+  "process.Is_Ready": "已就緒",
+  "process.Parsing": "內容解析中",
+  "process.Vectorizing": "索引向量化",
+  "process.Waiting": "排隊中",
  "rebuild_embedding_start_tip": "切換索引模型任務已開始",
  "rebuilding_index_count": "重建中索引數量：{{count}}",
  "request_headers": "請求頭",
@@ -114,7 +136,10 @@
  "tag.total_tags": "共 {{total}} 個標籤",
  "the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "資料集有索引正在訓練或重建中",
  "total_num_files": "共 {{total}} 個文件",
+  "training.Error": "{{count}} 組異常",
+  "training.Normal": "正常",
  "training_mode": "分段模式",
+  "training_ready": "{{count}} 組",
  "vector_model_max_tokens_tip": "每個分塊數據，最大長度為 3000 tokens",
  "vllm_model": "圖片理解模型",
  "website_dataset": "網站同步",