Feat: Images dataset collection (#4941)
* New pic (#4858) * 更新数据集相关类型,添加图像文件ID和预览URL支持;优化数据集导入功能,新增图像数据集处理组件;修复部分国际化文本;更新文件上传逻辑以支持新功能。 * 与原先代码的差别 * 新增 V4.9.10 更新说明,支持 PG 设置`systemEnv.hnswMaxScanTuples`参数,优化 LLM stream 调用超时,修复全文检索多知识库排序问题。同时更新数据集索引,移除 datasetId 字段以简化查询。 * 更换成fileId_image逻辑,并增加训练队列匹配的逻辑 * 新增图片集合判断逻辑,优化预览URL生成流程,确保仅在数据集为图片集合时生成预览URL,并添加相关日志输出以便调试。 * Refactor Docker Compose configuration to comment out exposed ports for production environments, update image versions for pgvector, fastgpt, and mcp_server, and enhance Redis service with a health check. Additionally, standardize dataset collection labels in constants and improve internationalization strings across multiple languages. * Enhance TrainingStates component by adding internationalization support for the imageParse training mode and update defaultCounts to include imageParse mode in trainingDetail API. * Enhance dataset import context by adding additional steps for image dataset import process and improve internationalization strings for modal buttons in the useEditTitle hook. * Update DatasetImportContext to conditionally render MyStep component based on data source type, improving the import process for non-image datasets. * Refactor image dataset handling by improving internationalization strings, enhancing error messages, and streamlining the preview URL generation process. * 图片上传到新建的 dataset_collection_images 表,逻辑跟随更改 * 修改了除了controller的其他部分问题 * 把图片数据集的逻辑整合到controller里面 * 补充i18n * 补充i18n * resolve评论:主要是上传逻辑的更改和组件复用 * 图片名称的图标显示 * 修改编译报错的命名问题 * 删除不需要的collectionid部分 * 多余文件的处理和改动一个删除按钮 * 除了loading和统一的imageId,其他都resolve掉的 * 处理图标报错 * 复用了MyPhotoView并采用全部替换的方式将imageFileId变成imageId * 去除不必要文件修改 * 报错和字段修改 * 增加上传成功后删除临时文件的逻辑以及回退一些修改 * 删除path字段,将图片保存到gridfs内,并修改增删等操作的代码 * 修正编译错误 --------- Co-authored-by: archer <545436317@qq.com> * perf: image dataset * feat: insert image * perf: image icon * fix: training state --------- Co-authored-by: Zhuangzai fa <143257420+ctrlz526@users.noreply.github.com>
This commit is contained in:
@@ -28,16 +28,21 @@
|
||||
"collection.training_type": "处理模式",
|
||||
"collection_data_count": "数据量",
|
||||
"collection_metadata_custom_pdf_parse": "PDF增强解析",
|
||||
"collection_name": "数据集名称",
|
||||
"collection_not_support_retraining": "该集合类型不支持重新调整参数",
|
||||
"collection_not_support_sync": "该集合不支持同步",
|
||||
"collection_sync": "立即同步",
|
||||
"collection_sync_confirm_tip": "确认开始同步数据?系统将会拉取最新数据进行比较,如果内容不相同,则会创建一个新的集合并删除旧的集合,请确认!",
|
||||
"collection_tags": "集合标签",
|
||||
"common.dataset.data.Input Error Tip": "[图片数据集] 处理过程错误:",
|
||||
"common.error.unKnow": "未知错误",
|
||||
"common_dataset": "通用知识库",
|
||||
"common_dataset_desc": "通过导入文件、网页链接或手动录入形式构建知识库",
|
||||
"condition": "条件",
|
||||
"config_sync_schedule": "配置定时同步",
|
||||
"confirm_import_images": "共 {{num}} 张图片 | 确认创建",
|
||||
"confirm_to_rebuild_embedding_tip": "确认为知识库切换索引?\n切换索引是一个非常重量的操作,需要对您知识库内所有数据进行重新索引,时间可能较长,请确保账号内剩余积分充足。\n\n此外,你还需要注意修改选择该知识库的应用,避免它们与其他索引模型知识库混用。",
|
||||
"core.dataset.Image collection": "图片数据集",
|
||||
"core.dataset.import.Adjust parameters": "调整参数",
|
||||
"custom_data_process_params": "自定义",
|
||||
"custom_data_process_params_desc": "自定义设置数据处理规则",
|
||||
@@ -90,6 +95,7 @@
|
||||
"image_auto_parse": "图片自动索引",
|
||||
"image_auto_parse_tips": "调用 VLM 自动标注文档里的图片,并生成额外的检索索引",
|
||||
"image_training_queue": "图片处理排队",
|
||||
"images_creating": "正在创建",
|
||||
"immediate_sync": "立即同步",
|
||||
"import.Auto mode Estimated Price Tips": "需调用文本理解模型,需要消耗较多AI 积分:{{price}} 积分/1K tokens",
|
||||
"import.Embedding Estimated Price Tips": "仅使用索引模型,消耗少量 AI 积分:{{price}} 积分/1K tokens",
|
||||
@@ -104,6 +110,8 @@
|
||||
"index_size": "索引大小",
|
||||
"index_size_tips": "向量化时内容的长度,系统会自动按该大小对分块进行进一步的分割。",
|
||||
"input_required_field_to_select_baseurl": "请先输入必填信息",
|
||||
"insert_images": "新增图片",
|
||||
"insert_images_success": "新增图片成功,需等待训练完成才会展示",
|
||||
"is_open_schedule": "启用定时同步",
|
||||
"keep_image": "保留图片",
|
||||
"loading": "加载中...",
|
||||
@@ -135,6 +143,7 @@
|
||||
"process.Image_Index": "图片索引生成",
|
||||
"process.Is_Ready": "已就绪",
|
||||
"process.Is_Ready_Count": "{{count}} 组已就绪",
|
||||
"process.Parse_Image": "图片解析中",
|
||||
"process.Parsing": "内容解析中",
|
||||
"process.Vectorizing": "索引向量化",
|
||||
"process.Waiting": "排队中",
|
||||
@@ -176,11 +185,14 @@
|
||||
"the_knowledge_base_has_indexes_that_are_being_trained_or_being_rebuilt": "知识库有训练中或正在重建的索引",
|
||||
"total_num_files": "共 {{total}} 个文件",
|
||||
"training.Error": "{{count}} 组异常",
|
||||
"training.Image mode": "图片处理",
|
||||
"training.Normal": "正常",
|
||||
"training_mode": "处理方式",
|
||||
"training_ready": "{{count}} 组",
|
||||
"uploading_progress": "上传中: {{num}}%",
|
||||
"vector_model_max_tokens_tip": "每个分块数据,最大长度为 3000 tokens",
|
||||
"vllm_model": "图片理解模型",
|
||||
"vlm_model_required_warning": "需要图片理解模型",
|
||||
"website_dataset": "Web 站点同步",
|
||||
"website_dataset_desc": "通过爬虫,批量爬取网页数据构建知识库",
|
||||
"website_info": "网站信息",
|
||||
|
||||
Reference in New Issue
Block a user