website sync feature (#4429)

* perf: introduce BullMQ for website sync (#4403)

* perf: introduce BullMQ for website sync

* feat: new redis module

* fix: remove graceful shutdown

* perf: improve UI in dataset detail

- Updated the "change" icon SVG file.
- Modified i18n strings.
- Added new i18n string "immediate_sync".
- Improved UI in dataset detail page, including button icons and
background colors.

* refactor: Add chunkSettings to DatasetSchema

* perf: website sync ux

* env template

* fix: clean up website dataset when updating chunk settings (#4420)

* perf: check setting updated

* perf: worker currency

* feat: init script for website sync refactor (#4425)

* website feature doc

---------

Co-authored-by: a.e. <49438478+I-Info@users.noreply.github.com>

packages/service/common/bullmq/index.ts

@@ -0,0 +1,74 @@
+import { ConnectionOptions, Processor, Queue, QueueOptions, Worker, WorkerOptions } from 'bullmq';
+import { addLog } from '../system/log';
+import { newQueueRedisConnection, newWorkerRedisConnection } from '../redis';
+
+const defaultWorkerOpts: Omit<ConnectionOptions, 'connection'> = {
+  removeOnComplete: {
+    count: 0 // Delete jobs immediately on completion
+  },
+  removeOnFail: {
+    count: 0 // Delete jobs immediately on failure
+  }
+};
+
+export enum QueueNames {
+  websiteSync = 'websiteSync'
+}
+
+export const queues = (() => {
+  if (!global.queues) {
+    global.queues = new Map<QueueNames, Queue>();
+  }
+  return global.queues;
+})();
+export const workers = (() => {
+  if (!global.workers) {
+    global.workers = new Map<QueueNames, Worker>();
+  }
+  return global.workers;
+})();
+
+export function getQueue<DataType, ReturnType = void>(
+  name: QueueNames,
+  opts?: Omit<QueueOptions, 'connection'>
+): Queue<DataType, ReturnType> {
+  // check if global.queues has the queue
+  const queue = queues.get(name);
+  if (queue) {
+    return queue as Queue<DataType, ReturnType>;
+  }
+  const newQueue = new Queue<DataType, ReturnType>(name.toString(), {
+    connection: newQueueRedisConnection(),
+    ...opts
+  });
+
+  // default error handler, to avoid unhandled exceptions
+  newQueue.on('error', (error) => {
+    addLog.error(`MQ Queue [${name}]: ${error.message}`, error);
+  });
+  queues.set(name, newQueue);
+  return newQueue;
+}
+
+export function getWorker<DataType, ReturnType = void>(
+  name: QueueNames,
+  processor: Processor<DataType, ReturnType>,
+  opts?: Omit<WorkerOptions, 'connection'>
+): Worker<DataType, ReturnType> {
+  const worker = workers.get(name);
+  if (worker) {
+    return worker as Worker<DataType, ReturnType>;
+  }
+
+  const newWorker = new Worker<DataType, ReturnType>(name.toString(), processor, {
+    connection: newWorkerRedisConnection(),
+    ...defaultWorkerOpts,
+    ...opts
+  });
+  // default error handler, to avoid unhandled exceptions
+  newWorker.on('error', (error) => {
+    addLog.error(`MQ Worker [${name}]: ${error.message}`, error);
+  });
+  workers.set(name, newWorker);
+  return newWorker;
+}

packages/service/common/bullmq/type.d.ts

@@ -0,0 +1,7 @@
+import { Queue, Worker } from 'bullmq';
+import { QueueNames } from './index';
+
+declare global {
+  var queues: Map<QueueNames, Queue> | undefined;
+  var workers: Map<QueueNames, Worker> | undefined;
+}

packages/service/common/redis/index.ts

@@ -0,0 +1,27 @@
+import Redis from 'ioredis';
+
+const REDIS_URL = process.env.REDIS_URL ?? 'redis://localhost:6379';
+
+export function newQueueRedisConnection() {
+  const redis = new Redis(REDIS_URL);
+  redis.on('connect', () => {
+    console.log('Redis connected');
+  });
+  redis.on('error', (error) => {
+    console.error('Redis connection error', error);
+  });
+  return redis;
+}
+
+export function newWorkerRedisConnection() {
+  const redis = new Redis(REDIS_URL, {
+    maxRetriesPerRequest: null
+  });
+  redis.on('connect', () => {
+    console.log('Redis connected');
+  });
+  redis.on('error', (error) => {
+    console.error('Redis connection error', error);
+  });
+  return redis;
+}