website sync feature (#4429)

* perf: introduce BullMQ for website sync (#4403)

* perf: introduce BullMQ for website sync

* feat: new redis module

* fix: remove graceful shutdown

* perf: improve UI in dataset detail

- Updated the "change" icon SVG file.
- Modified i18n strings.
- Added new i18n string "immediate_sync".
- Improved UI in dataset detail page, including button icons and
background colors.

* refactor: Add chunkSettings to DatasetSchema

* perf: website sync ux

* env template

* fix: clean up website dataset when updating chunk settings (#4420)

* perf: check setting updated

* perf: worker currency

* feat: init script for website sync refactor (#4425)

* website feature doc

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

packages/web/i18n/en/common.json

@@ -512,7 +512,7 @@
   "core.dataset.Query extension intro": "Enabling the question optimization function can improve the accuracy of Dataset searches during continuous conversations. After enabling this function, when performing Dataset searches, the AI will complete the missing information of the question based on the conversation history.",
   "core.dataset.Quote Length": "Quote Content Length",
   "core.dataset.Read Dataset": "View Dataset Details",
-  "core.dataset.Set Website Config": "Start Configuring Website Information",
+  "core.dataset.Set Website Config": "Start Configuring",
   "core.dataset.Start export": "Export Started",
   "core.dataset.Table collection": "Table Dataset",
   "core.dataset.Text collection": "Text Dataset",
@@ -528,7 +528,6 @@
   "core.dataset.collection.Website Empty Tip": "No Website Associated Yet",
   "core.dataset.collection.Website Link": "Website Address",
   "core.dataset.collection.id": "Collection ID",
-  "core.dataset.collection.metadata.Chunk Size": "Chunk Size",
   "core.dataset.collection.metadata.Createtime": "Creation Time",
   "core.dataset.collection.metadata.Raw text length": "Raw Text Length",
   "core.dataset.collection.metadata.Updatetime": "Update Time",
@@ -630,6 +629,7 @@
   "core.dataset.search.search mode": "Search Method",
   "core.dataset.status.active": "Ready",
   "core.dataset.status.syncing": "Syncing",
+  "core.dataset.status.waiting": "Waiting",
   "core.dataset.test.Batch test": "Batch Test",
   "core.dataset.test.Batch test Placeholder": "Select a CSV File",
   "core.dataset.test.Search Test": "Search Test",

packages/web/i18n/en/dataset.json

@@ -7,6 +7,7 @@
   "auto_indexes_tips": "Additional index generation is performed through large models to improve semantic richness and improve retrieval accuracy.",
   "auto_training_queue": "Enhanced index queueing",
   "chunk_max_tokens": "max_tokens",
+  "chunk_size": "Block size",
   "close_auto_sync": "Are you sure you want to turn off automatic sync?",
   "collection.Create update time": "Creation/Update Time",
   "collection.Training type": "Training",
@@ -70,6 +71,7 @@
   "image_auto_parse": "Automatic image indexing",
   "image_auto_parse_tips": "Call VLM to automatically label the pictures in the document and generate additional search indexes",
   "image_training_queue": "Queue of image processing",
+  "immediate_sync": "Immediate Synchronization",
   "import.Auto mode Estimated Price Tips": "The text understanding model needs to be called, which requires more points: {{price}} points/1K tokens",
   "import.Embedding Estimated Price Tips": "Only use the index model and consume a small amount of AI points: {{price}} points/1K tokens",
   "import_confirm": "Confirm upload",
@@ -86,6 +88,7 @@
   "keep_image": "Keep the picture",
   "move.hint": "After moving, the selected knowledge base/folder will inherit the permission settings of the new folder, and the original permission settings will become invalid.",
   "open_auto_sync": "After scheduled synchronization is turned on, the system will try to synchronize the collection from time to time every day. During the collection synchronization period, the collection data will not be searched.",
+  "params_config": "Config",
   "params_setting": "Parameter settings",
   "pdf_enhance_parse": "PDF enhancement analysis",
   "pdf_enhance_parse_price": "{{price}} points/page",
@@ -144,6 +147,7 @@
   "vllm_model": "Image understanding model",
   "website_dataset": "Website Sync",
   "website_dataset_desc": "Website sync allows you to build a Dataset directly using a web link.",
+  "website_info": "Website Information",
   "yuque_dataset": "Yuque Dataset",
   "yuque_dataset_config": "Yuque Dataset Config",
   "yuque_dataset_desc": "Can build a dataset using Yuque documents by configuring permissions, without secondary storage"