feat: add websocket support for real-time scrape job monitoring

src/scraper/mod.rs

@@ -7,6 +7,7 @@ use crate::data::scrape_jobs;
 use crate::services::Service;
 use crate::state::ReferenceCache;
 use crate::status::{ServiceStatus, ServiceStatusRegistry};
+use crate::web::ws::ScrapeJobEvent;
 use sqlx::PgPool;
 use std::sync::Arc;
 use tokio::sync::{RwLock, broadcast};
@@ -25,6 +26,7 @@ pub struct ScraperService {
     banner_api: Arc<BannerApi>,
     reference_cache: Arc<RwLock<ReferenceCache>>,
     service_statuses: ServiceStatusRegistry,
+    job_events_tx: broadcast::Sender<ScrapeJobEvent>,
     scheduler_handle: Option<JoinHandle<()>>,
     worker_handles: Vec<JoinHandle<()>>,
     shutdown_tx: Option<broadcast::Sender<()>>,
@@ -37,12 +39,14 @@ impl ScraperService {
         banner_api: Arc<BannerApi>,
         reference_cache: Arc<RwLock<ReferenceCache>>,
         service_statuses: ServiceStatusRegistry,
+        job_events_tx: broadcast::Sender<ScrapeJobEvent>,
     ) -> Self {
         Self {
             db_pool,
             banner_api,
             reference_cache,
             service_statuses,
+            job_events_tx,
             scheduler_handle: None,
             worker_handles: Vec::new(),
             shutdown_tx: None,
@@ -71,6 +75,7 @@ impl ScraperService {
             self.db_pool.clone(),
             self.banner_api.clone(),
             self.reference_cache.clone(),
+            self.job_events_tx.clone(),
         );
         let shutdown_rx = shutdown_tx.subscribe();
         let scheduler_handle = tokio::spawn(async move {
@@ -81,7 +86,12 @@ impl ScraperService {
 
         let worker_count = 4; // This could be configurable
         for i in 0..worker_count {
-            let worker = Worker::new(i, self.db_pool.clone(), self.banner_api.clone());
+            let worker = Worker::new(
+                i,
+                self.db_pool.clone(),
+                self.banner_api.clone(),
+                self.job_events_tx.clone(),
+            );
             let shutdown_rx = shutdown_tx.subscribe();
             let worker_handle = tokio::spawn(async move {
                 worker.run(shutdown_rx).await;