Feat: Images dataset collection (#4941)

* New pic (#4858)

* 更新数据集相关类型，添加图像文件ID和预览URL支持；优化数据集导入功能，新增图像数据集处理组件；修复部分国际化文本；更新文件上传逻辑以支持新功能。

* 与原先代码的差别

* 新增 V4.9.10 更新说明，支持 PG 设置`systemEnv.hnswMaxScanTuples`参数，优化 LLM stream 调用超时，修复全文检索多知识库排序问题。同时更新数据集索引，移除 datasetId 字段以简化查询。

* 更换成fileId_image逻辑，并增加训练队列匹配的逻辑

* 新增图片集合判断逻辑，优化预览URL生成流程，确保仅在数据集为图片集合时生成预览URL，并添加相关日志输出以便调试。

* Refactor Docker Compose configuration to comment out exposed ports for production environments, update image versions for pgvector, fastgpt, and mcp_server, and enhance Redis service with a health check. Additionally, standardize dataset collection labels in constants and improve internationalization strings across multiple languages.

* Enhance TrainingStates component by adding internationalization support for the imageParse training mode and update defaultCounts to include imageParse mode in trainingDetail API.

* Enhance dataset import context by adding additional steps for image dataset import process and improve internationalization strings for modal buttons in the useEditTitle hook.

* Update DatasetImportContext to conditionally render MyStep component based on data source type, improving the import process for non-image datasets.

* Refactor image dataset handling by improving internationalization strings, enhancing error messages, and streamlining the preview URL generation process.

* 图片上传到新建的 dataset_collection_images 表，逻辑跟随更改

* 修改了除了controller的其他部分问题

* 把图片数据集的逻辑整合到controller里面

* 补充i18n

* 补充i18n

* resolve评论：主要是上传逻辑的更改和组件复用

* 图片名称的图标显示

* 修改编译报错的命名问题

* 删除不需要的collectionid部分

* 多余文件的处理和改动一个删除按钮

* 除了loading和统一的imageId,其他都resolve掉的

* 处理图标报错

* 复用了MyPhotoView并采用全部替换的方式将imageFileId变成imageId

* 去除不必要文件修改

* 报错和字段修改

* 增加上传成功后删除临时文件的逻辑以及回退一些修改

* 删除path字段，将图片保存到gridfs内，并修改增删等操作的代码

* 修正编译错误

---------

Co-authored-by: archer <545436317@qq.com>

* perf: image dataset

* feat: insert image

* perf: image icon

* fix: training state

---------

Co-authored-by: Zhuangzai fa <143257420+ctrlz526@users.noreply.github.com>

packages/web/i18n/en/chat.json

@@ -71,13 +71,13 @@
   "response_embedding_model_tokens": "Vector Model Tokens",
   "response_hybrid_weight": "Embedding : Full text = {{emb}} : {{text}}",
   "response_rerank_tokens": "Rearrange Model Tokens",
+  "search_results": "Search results",
   "select": "Select",
   "select_file": "Upload File",
   "select_file_img": "Upload file / image",
   "select_img": "Upload Image",
   "source_cronJob": "Scheduled execution",
   "stream_output": "Stream Output",
-  "to_dataset": "Go to the Knowledge Base",
   "unsupported_file_type": "Unsupported file types",
   "upload": "Upload",
   "variable_invisable_in_share": "Custom variables are not visible in login-free links",

packages/web/i18n/en/common.json

@@ -180,7 +180,7 @@
   "code_error.user_error.balance_not_enough": "Insufficient Account Balance",
   "code_error.user_error.bin_visitor_guest": "You Are Currently a Guest, Unauthorized to Operate",
   "code_error.user_error.un_auth_user": "User Not Found",
-  "comfirm_import": "comfirm_import",
+  "comfirm_import": "Confirm import",
   "comfirm_leave_page": "Confirm to Leave This Page?",
   "comfirn_create": "Confirm Creation",
   "commercial_function_tip": "Please Upgrade to the Commercial Version to Use This Feature: https://doc.fastgpt.cn/docs/commercial/intro/",
@@ -403,7 +403,6 @@
   "core.chat.response.module model": "Model",
   "core.chat.response.module name": "Model Name",
   "core.chat.response.module query": "Question/Search Term",
-  "core.chat.response.module quoteList": "Quote Content",
   "core.chat.response.module similarity": "Similarity",
   "core.chat.response.module temperature": "Temperature",
   "core.chat.response.module time": "Run Time",
@@ -434,7 +433,6 @@
   "core.dataset.Text collection": "Text Dataset",
   "core.dataset.apiFile": "API File",
   "core.dataset.collection.Click top config website": "Click to Configure Website",
-  "core.dataset.collection.Collection name": "Dataset Name",
   "core.dataset.collection.Collection raw text": "Dataset Content",
   "core.dataset.collection.Empty Tip": "The Dataset is Empty",
   "core.dataset.collection.QA Prompt": "QA Split Prompt",
@@ -451,7 +449,6 @@
   "core.dataset.collection.metadata.metadata": "Metadata",
   "core.dataset.collection.metadata.read source": "View Original Content",
   "core.dataset.collection.metadata.source": "Data Source",
-  "core.dataset.collection.metadata.source name": "Source Name",
   "core.dataset.collection.metadata.source size": "Source Size",
   "core.dataset.collection.status.active": "Ready",
   "core.dataset.collection.status.error": "Error",
@@ -743,7 +740,7 @@
   "core.workflow.value": "Value",
   "core.workflow.variable": "Variable",
   "create": "Create",
-  "create_failed": "Creation Failed",
+  "create_failed": "Create failed",
   "create_success": "Created Successfully",
   "create_time": "Creation Time",
   "cron_job_run_app": "Scheduled Task",
@@ -788,7 +785,6 @@
   "dataset.dataset_name": "Dataset Name",
   "dataset.deleteFolderTips": "Confirm to Delete This Folder and All Its Contained Datasets? Data Cannot Be Recovered After Deletion, Please Confirm!",
   "dataset.test.noResult": "No Search Results",
-  "dataset_data_import_q_placeholder": "Up to {{maxToken}} words.",
   "dataset_data_input_a": "Answer",
   "dataset_data_input_chunk": "Chunk",
   "dataset_data_input_chunk_content": "Chunk",
@@ -802,7 +798,6 @@
   "delete_success": "Deleted Successfully",
   "delete_warning": "Deletion Warning",
   "embedding_model_not_config": "No index model is detected",
-  "error.Create failed": "Create failed",
   "error.code_error": "Verification code error",
   "error.fileNotFound": "File not found~",
   "error.inheritPermissionError": "Inherit permission Error",
@@ -1208,6 +1203,7 @@
   "templateTags.Writing": "Writing",
   "template_market": "Template Market",
   "textarea_variable_picker_tip": "Enter \"/\" to select a variable",
+  "to_dataset": "To dataset",
   "ui.textarea.Magnifying": "Magnifying",
   "un_used": "Unused",
   "unauth_token": "The certificate has expired, please log in again",

packages/web/i18n/en/dataset.json

@@ -28,16 +28,21 @@
   "collection.training_type": "Chunk type",
   "collection_data_count": "Data amount",
   "collection_metadata_custom_pdf_parse": "PDF enhancement analysis",
+  "collection_name": "Collection name",
   "collection_not_support_retraining": "This collection type does not support retuning parameters",
   "collection_not_support_sync": "This collection does not support synchronization",
   "collection_sync": "Sync data",
   "collection_sync_confirm_tip": "Confirm to start synchronizing data? \nThe system will pull the latest data for comparison. If the contents are different, a new collection will be created and the old collection will be deleted. Please confirm!",
   "collection_tags": "Collection Tags",
+  "common.dataset.data.Input Error Tip": "[Image Dataset] Process error:",
+  "common.error.unKnow": "Unknown error",
   "common_dataset": "General Dataset",
   "common_dataset_desc": "Building a knowledge base by importing files, web page links, or manual entry",
   "condition": "condition",
   "config_sync_schedule": "Configure scheduled synchronization",
+  "confirm_import_images": "Total {{num}} | Confirm create",
   "confirm_to_rebuild_embedding_tip": "Are you sure you want to switch the index for the Dataset?\nSwitching the index is a significant operation that requires re-indexing all data in your Dataset, which may take a long time. Please ensure your account has sufficient remaining points.\n\nAdditionally, you need to update the applications that use this Dataset to avoid conflicts with other indexed model Datasets.",
+  "core.dataset.Image collection": "Image dataset",
   "core.dataset.import.Adjust parameters": "Adjust parameters",
   "custom_data_process_params": "Custom",
   "custom_data_process_params_desc": "Customize data processing rules",
@@ -90,6 +95,7 @@
   "image_auto_parse": "Automatic image indexing",
   "image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes",
   "image_training_queue": "Queue of image processing",
+  "images_creating": "Creating",
   "immediate_sync": "Immediate Synchronization",
   "import.Auto mode Estimated Price Tips": "The text understanding model needs to be called, which requires more points: {{price}} points/1K tokens",
   "import.Embedding Estimated Price Tips": "Only use the index model and consume a small amount of AI points: {{price}} points/1K tokens",
@@ -104,6 +110,8 @@
   "index_size": "Index size",
   "index_size_tips": "When vectorized, the system will automatically further segment the blocks according to this size.",
   "input_required_field_to_select_baseurl": "Please enter the required information first",
+  "insert_images": "Added pictures",
+  "insert_images_success": "The new picture is successfully added, and you need to wait for the training to be completed before it will be displayed.",
   "is_open_schedule": "Enable scheduled synchronization",
   "keep_image": "Keep the picture",
   "loading": "Loading...",
@@ -135,6 +143,7 @@
   "process.Image_Index": "Image index generation",
   "process.Is_Ready": "Ready",
   "process.Is_Ready_Count": "{{count}} Group is ready",
+  "process.Parse_Image": "Image analysis",
   "process.Parsing": "Parsing",
   "process.Vectorizing": "Index vectorization",
   "process.Waiting": "Queue",
@@ -179,13 +188,19 @@
   "training.Error": "{{count}} Group exception",
   "training.Normal": "Normal",
   "training_mode": "Chunk mode",
+  "training_queue_tip": "Training queue status",
   "training_ready": "{{count}} Group",
+  "uploading_progress": "Uploading: {{num}}%",
   "vector_model_max_tokens_tip": "Each chunk of data has a maximum length of 3000 tokens",
+  "vector_training_queue": "Vector training queue",
   "vllm_model": "Image understanding model",
+  "vlm_model_required_tooltip": "A Vision Language Model is required to create image collections",
+  "vlm_model_required_warning": "Image datasets require a Vision Language Model (VLM) to be configured. Please add a model that supports image understanding in the model configuration first.",
+  "waiting_for_training": "Waiting for training",
   "website_dataset": "Website Sync",
   "website_dataset_desc": "Build knowledge base by crawling web page data in batches",
   "website_info": "Website Information",
-  "yuque_dataset": "Yuque Dataset",
-  "yuque_dataset_config": "Yuque Dataset Config",
-  "yuque_dataset_desc": "Can build a dataset using Yuque documents by configuring permissions, without secondary storage"
+  "yuque_dataset": "Yuque Knowledge Base",
+  "yuque_dataset_config": "Configure Yuque Knowledge Base",
+  "yuque_dataset_desc": "Build knowledge base using Yuque documents by configuring document permissions, documents will not be stored twice"
 }

packages/web/i18n/en/file.json

@@ -1,9 +1,32 @@
 {
+  "Action": "Please select the image to upload",
+  "All images import failed": "All pictures failed to import",
+  "Dataset_ID_not_found": "The dataset ID does not exist",
+  "Failed_to_get_token": "Failed to obtain the token",
+  "Image_ID_copied": "Copy ID",
+  "Image_Preview": "Picture preview",
+  "Image_dataset_requires_VLM_model_to_be_configured": "The image dataset needs to be configured with the image understanding model (VLM) to be used. Please add a model that supports image understanding in the model configuration first.",
+  "Image_does_not_belong_to_current_team": "The picture does not belong to the current team",
+  "Image_file_does_not_exist": "The picture does not exist",
+  "Loading_image": "Loading the picture...",
+  "Loading_image failed": "Preview loading failed",
+  "Only_support_uploading_one_image": "Only support uploading one image",
+  "Please select the image to upload": "Please select the image to upload",
+  "Please select the image to upload select the image to upload": "",
+  "Please wait for all files to upload": "Please wait for all files to be uploaded to complete",
   "bucket_chat": "Conversation Files",
   "bucket_file": "Dataset Documents",
   "click_to_view_raw_source": "Click to View Original Source",
+  "common.dataset_data_input_image_support_format": "Support .jpg, .jpeg, .png, .gif, .webp formats",
+  "delete_image": "Delete pictures",
   "file_name": "Filename",
   "file_size": "Filesize",
+  "image": "picture",
+  "image_collection": "Picture collection",
+  "image_description": "Image description",
+  "image_description_tip": "Please enter the description of the picture",
+  "please_upload_image_first": "Please upload the picture first",
+  "reached_max_file_count": "Maximum file count reached",
   "release_the_mouse_to_upload_the_file": "Release Mouse to Upload File",
   "select_and_drag_file_tip": "Click or Drag Files Here to Upload",
   "select_file_amount_limit": "You can select up to {{max}} files",
@@ -12,7 +35,9 @@
   "support_file_type": "Supports {{fileType}} file types",
   "support_max_count": "Supports up to {{maxCount}} files",
   "support_max_size": "Maximum file size is {{maxSize}}",
+  "total_files": "Total {{selectFiles.length}} files",
+  "upload_error_description": "Only multiple files or a single folder can be uploaded at a time",
   "upload_failed": "Upload Failed",
-  "reached_max_file_count": "Maximum file count reached",
-  "upload_error_description": "Only multiple files or a single folder can be uploaded at a time"
-}
+  "upload_file_error": "Please upload pictures",
+  "uploading": "Uploading..."
+}