Merge pull request #945 from eetu/pv-forecast

Add 66h PV forecast overlay

Author: eetu

README.md

@@ -22,8 +22,13 @@ HCC_HISTORY_RETENTION_DAYS=90 # days of sensor history to keep (default: 0/disab
 # Maps room types to lists of room names as configured in the Hue app.
 # Rooms not listed default to "inside".
 HUE_ROOM_TYPES={"inside": ["Keittiö", "Olohuone"], "inside_cold": ["Kuisti"], "outside": ["Ulkona"]}
+
 ```
 
+PV forecast vars (`PV_LAT`, `PV_LON`, `PV_TILT`, `PV_AZIMUTH`, `PV_KW`) are
+consumed by `scripts/refresh-pv-forecast.sh` — see the "PV forecast"
+section below for details.
+
 ### Pairing with the Hue bridge
 
 1. Press the link button on the bridge
@@ -69,6 +74,56 @@ GET /api/history/sensors?sensor_id=<id>&hours=<n>
 | `sensor_id` | no       | all     | Filter to a specific sensor          |
 | `hours`     | no       | 24      | Hours of history to return (max 720) |
 
+## PV forecast
+
+The backend exposes `GET/POST /api/pv/forecast`. The actual forecast is
+produced by an external one-shot CLI maintained in a separate repository,
+[fmi-pv-forecast-runner](https://github.com/eetu/fmi-pv-forecast-runner),
+which wraps the
+[FMI open PV forecast](https://github.com/fmidev/fmi-open-pv-forecast-packaged)
+package and emits the next ~66 hours of hourly output as JSON on stdout.
+
+Refreshing the forecast is a two-step shell pipeline: run the CLI, pipe
+its stdout to `POST /api/pv/forecast`. `scripts/refresh-pv-forecast.sh`
+wraps both:
+
+```bash
+# Auto-detect: uses local uv repo if ../fmi-pv-forecast-runner/ is checked out,
+# otherwise pulls the published Docker image.
+HCC_PV_ENV_FILE=.env.pv ./scripts/refresh-pv-forecast.sh
+
+# Force a specific mode:
+./scripts/refresh-pv-forecast.sh uv
+./scripts/refresh-pv-forecast.sh docker
+```
+
+Required env vars (place in `$HCC_PV_ENV_FILE` or export beforehand):
+`PV_LAT`, `PV_LON`, `PV_TILT`, `PV_AZIMUTH`, `PV_KW`. Optional overrides:
+`HCC_BACKEND_URL` (default `http://localhost:3000`), `PV_RUNNER_PATH`,
+`PV_RUNNER_IMAGE`.
+
+### Cold-start and periodic refresh
+
+Run the script once manually to populate the database after the first
+deploy. For a 3-hour refresh cadence (matching the upstream forecast
+update interval), add a host cron entry:
+
+```cron
+17 */3 * * * cd /opt/hcc && HCC_PV_ENV_FILE=/opt/hcc/.env.pv ./scripts/refresh-pv-forecast.sh >>/var/log/hcc-pv.log 2>&1
+```
+
+### Storage and rendering
+
+Hourly forecast points are upserted into the `pv_forecast_points` table
+(steady-state ≤66 rows) and rendered as bars on the daily weather chart,
+aggregated to kWh per day.
+
+### Geographic limits
+
+The FMI forecast covers Finland, Scandinavia, and the Baltic countries. See
+[ilmatieteenlaitos.fi/numerical-weather-prediction](https://en.ilmatieteenlaitos.fi/numerical-weather-prediction)
+for the full available area.
+
 ## Media
 
 ![screenshot](/documentation/screenshot.png)

backend/src/hue/client.rs

@@ -13,6 +13,13 @@ pub enum HueError {
     Http(#[from] reqwest::Error),
     #[error("Bridge error {status}: {body}")]
     Bridge { status: u16, body: String },
+    #[error("Failed to decode response from {path}: {source} (body: {body})")]
+    Decode {
+        path: String,
+        body: String,
+        #[source]
+        source: serde_json::Error,
+    },
 }
 
 pub async fn hue_fetch<T: serde::de::DeserializeOwned>(
@@ -40,10 +47,32 @@ pub async fn hue_fetch<T: serde::de::DeserializeOwned>(
     if !res.status().is_success() {
         let status = res.status().as_u16();
         let body = res.text().await.unwrap_or_default();
+        tracing::error!(path, status, body = %truncate(&body, 500), "Hue bridge returned non-success");
         return Err(HueError::Bridge { status, body });
     }
 
-    Ok(res.json().await?)
+    let body = res.text().await?;
+    serde_json::from_str::<HueList<T>>(&body).map_err(|e| {
+        tracing::error!(
+            path,
+            error = %e,
+            body = %truncate(&body, 500),
+            "Failed to decode Hue response"
+        );
+        HueError::Decode {
+            path: path.to_string(),
+            body,
+            source: e,
+        }
+    })
+}
+
+fn truncate(s: &str, n: usize) -> String {
+    if s.len() <= n {
+        s.to_string()
+    } else {
+        format!("{}…[+{} bytes]", &s[..n], s.len() - n)
+    }
 }
 
 pub async fn hue_put(

backend/src/hue/data.rs

@@ -36,7 +36,11 @@ async fn fetch_from_bridge(state: &Arc<AppState>) -> Result<HueResponse, HueErro
     let battery_by_device: HashMap<&str, u8> = device_powers
         .data
         .iter()
-        .filter_map(|dp| dp.power_state.battery_level.map(|b| (dp.owner.rid.as_str(), b)))
+        .filter_map(|dp| {
+            dp.power_state
+                .battery_level
+                .map(|b| (dp.owner.rid.as_str(), b))
+        })
         .collect();
 
     // device ID → room
@@ -65,7 +69,7 @@ async fn fetch_from_bridge(state: &Arc<AppState>) -> Result<HueResponse, HueErro
         .map(|m| {
             let (motion, updated_at) = match &m.motion.motion_report {
                 Some(report) => (report.motion, Some(report.changed.as_str())),
-                None => (m.motion.motion, None),
+                None => (m.motion.motion.unwrap_or(false), None),
             };
             (m.owner.rid.as_str(), (motion, updated_at))
         })
@@ -108,19 +112,17 @@ async fn fetch_from_bridge(state: &Arc<AppState>) -> Result<HueResponse, HueErro
                 .temperature_report
                 .as_ref()
                 .map(|r| r.temperature)
-                .unwrap_or(temp.temperature.temperature);
+                .or(temp.temperature.temperature);
 
-            let battery = battery_by_device
-                .get(temp.owner.rid.as_str())
-                .copied();
+            let battery = battery_by_device.get(temp.owner.rid.as_str()).copied();
 
             let motion_data = motion_by_device.get(temp.owner.rid.as_str());
 
             Sensor {
                 id: temp.id.clone(),
                 device_id: temp.owner.rid.clone(),
                 name,
-                temperature: Some(temperature),
+                temperature,
                 room_type,
                 enabled: temp.enabled,
                 battery,

backend/src/hue/handlers.rs

@@ -43,13 +43,17 @@ pub async fn get_data(state: web::Data<Arc<AppState>>) -> HttpResponse {
         (status = 200, description = "Server-sent events stream of Hue device changes")
     )
 )]
-pub async fn events_sse(state: web::Data<Arc<AppState>>) -> sse::Sse<impl futures_util::Stream<Item = Result<sse::Event, std::convert::Infallible>>> {
+pub async fn events_sse(
+    state: web::Data<Arc<AppState>>,
+) -> sse::Sse<impl futures_util::Stream<Item = Result<sse::Event, std::convert::Infallible>>> {
     let rx = subscribe(&state.hue_events_tx);
     let stream = BroadcastStream::new(rx).filter_map(|result| async move {
         match result {
             Ok(event) => {
                 let json = serde_json::to_string(&event).ok()?;
-                Some(Ok::<_, std::convert::Infallible>(sse::Event::Data(sse::Data::new(json))))
+                Some(Ok::<_, std::convert::Infallible>(sse::Event::Data(
+                    sse::Data::new(json),
+                )))
             }
             Err(_) => None,
         }
@@ -86,10 +90,7 @@ pub struct PairResponse {
         (status = 400, description = "Pairing failed")
     )
 )]
-pub async fn pair(
-    state: web::Data<Arc<AppState>>,
-    body: web::Json<PairRequest>,
-) -> HttpResponse {
+pub async fn pair(state: web::Data<Arc<AppState>>, body: web::Json<PairRequest>) -> HttpResponse {
     let bridge_ip = body
         .bridge_ip
         .clone()
@@ -114,16 +115,14 @@ pub async fn pair(
     let res = match state.hue_client.post(&url).json(&payload).send().await {
         Ok(r) => r,
         Err(e) => {
-            return HttpResponse::BadRequest()
-                .json(serde_json::json!({"error": e.to_string()}));
+            return HttpResponse::BadRequest().json(serde_json::json!({"error": e.to_string()}));
         }
     };
 
     let entries: Vec<serde_json::Value> = match res.json().await {
         Ok(v) => v,
         Err(e) => {
-            return HttpResponse::BadRequest()
-                .json(serde_json::json!({"error": e.to_string()}));
+            return HttpResponse::BadRequest().json(serde_json::json!({"error": e.to_string()}));
         }
     };
 
@@ -133,8 +132,7 @@ pub async fn pair(
             .get("description")
             .and_then(|d| d.as_str())
             .unwrap_or("Pairing failed");
-        return HttpResponse::BadRequest()
-            .json(serde_json::json!({"error": desc}));
+        return HttpResponse::BadRequest().json(serde_json::json!({"error": desc}));
     }
 
     if let Some(success) = entry.get("success") {
@@ -184,8 +182,7 @@ pub async fn toggle_group(
         Ok(res) => res,
         Err(e) => {
             tracing::error!("Failed to get group {group_id}: {e}");
-            return HttpResponse::BadGateway()
-                .json(serde_json::json!({"error": e.to_string()}));
+            return HttpResponse::BadGateway().json(serde_json::json!({"error": e.to_string()}));
         }
     };
 
@@ -202,8 +199,7 @@ pub async fn toggle_group(
         Ok(()) => HttpResponse::Ok().finish(),
         Err(e) => {
             tracing::error!("Failed to toggle group {group_id}: {e}");
-            HttpResponse::BadGateway()
-                .json(serde_json::json!({"error": e.to_string()}))
+            HttpResponse::BadGateway().json(serde_json::json!({"error": e.to_string()}))
         }
     }
 }

backend/src/hue/models.rs

@@ -84,7 +84,8 @@ pub struct TemperatureResource {
 
 #[derive(Debug, Deserialize)]
 pub struct TemperatureData {
-    pub temperature: f64,
+    /// Absent on disabled temperature sensors — only `temperature_valid` is emitted then.
+    pub temperature: Option<f64>,
     pub temperature_report: Option<TemperatureReport>,
 }
 
@@ -141,7 +142,8 @@ pub struct MotionResource {
 
 #[derive(Debug, Deserialize)]
 pub struct MotionData {
-    pub motion: bool,
+    /// Absent on disabled motion sensors — only `motion_valid` is emitted then.
+    pub motion: Option<bool>,
     pub motion_report: Option<MotionReport>,
 }
 

backend/src/lib.rs

@@ -1,5 +1,6 @@
 pub mod cache;
 pub mod hue;
+pub mod pv;
 pub mod settings;
 pub mod solis;
 pub mod storage;
@@ -25,6 +26,7 @@ pub struct AppState {
     pub tomorrow_cache: Cache<serde_json::Value>,
     pub weather_cache: Cache<weather::models::WeatherResponse>,
     pub solis_cache: Cache<solis::models::SolisWidgetData>,
+    pub pv_cache: Cache<pv::models::PvForecast>,
     pub hue_events_tx: broadcast::Sender<hue::events::HueLiveEvent>,
     pub storage: storage::Storage,
 }
@@ -39,6 +41,8 @@ pub struct AppState {
         hue::handlers::pair,
         hue::handlers::toggle_group,
         solis::handlers::get_data,
+        pv::handlers::get_forecast,
+        pv::handlers::post_forecast,
         status,
         sensor_history,
     ),
@@ -52,6 +56,8 @@ pub struct AppState {
         hue::handlers::PairRequest,
         hue::handlers::PairResponse,
         solis::models::SolisWidgetData,
+        pv::models::PvForecast,
+        pv::models::PvPoint,
         StatusResponse,
         storage::SensorReading,
     ))
@@ -187,6 +193,13 @@ pub fn create_app(
                 .service(
                     web::scope("/solis")
                         .route("", web::get().to(solis::handlers::get_data)),
+                )
+                .service(
+                    web::scope("/pv").service(
+                        web::resource("/forecast")
+                            .route(web::get().to(pv::handlers::get_forecast))
+                            .route(web::post().to(pv::handlers::post_forecast)),
+                    ),
                 ),
         )
         .service(SwaggerUi::new("/docs/{_:.*}").url("/api-doc/openapi.json", openapi))
@@ -230,6 +243,7 @@ pub fn create_test_app_state_with(settings: Settings) -> Arc<AppState> {
         tomorrow_cache: Cache::new(std::time::Duration::from_secs(3600)),
         weather_cache: Cache::new(std::time::Duration::from_secs(3600)),
         solis_cache: Cache::new(std::time::Duration::from_secs(300)),
+        pv_cache: Cache::new(std::time::Duration::from_secs(60)),
         hue_events_tx,
         storage,
     })
@@ -268,6 +282,7 @@ pub async fn run_server() -> std::io::Result<()> {
         tomorrow_cache: Cache::new(std::time::Duration::from_secs(3600)),
         weather_cache: Cache::new(std::time::Duration::from_secs(3600)),
         solis_cache: Cache::new(std::time::Duration::from_secs(300)),
+        pv_cache: Cache::new(std::time::Duration::from_secs(60)),
         hue_events_tx: hue_events_tx.clone(),
         storage,
     });

backend/src/pv/handlers.rs

@@ -0,0 +1,61 @@
+use std::sync::Arc;
+
+use actix_web::{web, HttpResponse};
+
+use super::models::PvForecast;
+use crate::AppState;
+
+const CACHE_KEY: &str = "pv";
+
+#[utoipa::path(
+    get,
+    path = "/api/pv/forecast",
+    responses(
+        (status = 200, description = "Latest PV forecast", body = super::models::PvForecast),
+        (status = 503, description = "No forecast available yet"),
+    )
+)]
+pub async fn get_forecast(state: web::Data<Arc<AppState>>) -> HttpResponse {
+    if let Some(cached) = state.pv_cache.get(CACHE_KEY).await {
+        return HttpResponse::Ok().json(cached);
+    }
+
+    match state.storage.read_pv_forecast().await {
+        Ok(Some(forecast)) => {
+            state.pv_cache.set(CACHE_KEY.into(), forecast.clone()).await;
+            HttpResponse::Ok().json(forecast)
+        }
+        Ok(None) => HttpResponse::ServiceUnavailable()
+            .json(serde_json::json!({"error": "no forecast available yet"})),
+        Err(e) => {
+            tracing::error!("Failed to read PV forecast: {e}");
+            HttpResponse::InternalServerError().finish()
+        }
+    }
+}
+
+#[utoipa::path(
+    post,
+    path = "/api/pv/forecast",
+    request_body = super::models::PvForecast,
+    responses(
+        (status = 200, description = "Forecast upserted"),
+    )
+)]
+pub async fn post_forecast(
+    state: web::Data<Arc<AppState>>,
+    body: web::Json<PvForecast>,
+) -> HttpResponse {
+    let forecast = body.into_inner();
+    match state.storage.upsert_pv_forecast(&forecast).await {
+        Ok(n) => {
+            state.pv_cache.set(CACHE_KEY.into(), forecast).await;
+            tracing::info!("Upserted {n} PV forecast points");
+            HttpResponse::Ok().json(serde_json::json!({"upserted": n}))
+        }
+        Err(e) => {
+            tracing::error!("Failed to upsert PV forecast: {e}");
+            HttpResponse::InternalServerError().finish()
+        }
+    }
+}

backend/src/pv/mod.rs

@@ -0,0 +1,3 @@
+pub mod handlers;
+pub mod models;
+pub mod storage;