From cc6f5800ddcd1b59aa9cc886c4389b6282fbd00e Mon Sep 17 00:00:00 2001 From: CI Date: Fri, 2 Oct 2026 08:57:12 +0100 Subject: [PATCH 1/2] fix(solar): build the PV10/PV90 band on the P50 it is returned with, and use the Open-Meteo ensemble spread pv_calibration() always built the 10%/90% band from the calibrated series and only consulted metric_pv_calibration_enable at its final return, where it swapped P50 back to the raw forecast. With the switch off the band and its centre line came from two different curves, so P10 could exceed P50 and P90 fall below it, and the P50-P10 gap the cloud model reads was arbitrary (#5345). - The band is now built from whichever series is returned as P50. With calibration off that is the raw forecast, still clipped to the array ceiling. - Open-Meteo's ensemble download was fetched and then overwritten, because every Open-Meteo branch asked for the band to be created. It is now kept: the ensemble's P10 and P90 are taken as ratios of the ensemble's own median and applied to the deterministic P50, and are scaled with P50 by calibration. The history-based band is still created when the ensemble download fails. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/predbat/solar_model.py | 28 +- apps/predbat/solcast.py | 137 ++++++++-- apps/predbat/tests/test_open_meteo.py | 47 ++-- apps/predbat/tests/test_solar_model.py | 26 +- apps/predbat/tests/test_solcast.py | 352 +++++++++++++++++++++++++ docs/apps-yaml.md | 6 +- tools/debug-journal.md | 2 +- 7 files changed, 533 insertions(+), 65 deletions(-) diff --git a/apps/predbat/solar_model.py b/apps/predbat/solar_model.py index 9b39508af..a19d673c9 100644 --- a/apps/predbat/solar_model.py +++ b/apps/predbat/solar_model.py @@ -35,6 +35,7 @@ # Applied when no ensemble P10 data is available _DEFAULT_P10_FALLBACK = 0.7 +_DEFAULT_P90_FALLBACK = 1.3 def pvwatts_cell_temperature(poa_global, temp_air, wind_speed): @@ -68,7 +69,7 @@ def _temperature_efficiency(gti, temp, wind): return max(0.5, min(1.1, 1.0 - _TEMP_COEFF * (t_cell - _STC_TEMP_C))) -def gti_hourly_to_period_kwh(times, gti_values, temp_values, wind_values, kwp, system_loss, shading_factors=None, p10_instant=None, p10_fallback=_DEFAULT_P10_FALLBACK): +def gti_hourly_to_period_kwh(times, gti_values, temp_values, wind_values, kwp, system_loss, shading_factors=None, band_ratio=None, p10_fallback=_DEFAULT_P10_FALLBACK, p90_fallback=_DEFAULT_P90_FALLBACK): """Convert hourly GTI samples into per-hour PV energy for a single array. Open-Meteo returns point-in-time irradiance (W/m2) at the start of each hour, so the @@ -83,11 +84,13 @@ def gti_hourly_to_period_kwh(times, gti_values, temp_values, wind_values, kwp, s kwp: array peak power in kW system_loss: fractional system loss, e.g. 0.05 for 95% efficiency shading_factors: optional list of 12 per-month multipliers - p10_instant: optional dict of timestamp string to raw P10 kW, before temperature derate - p10_fallback: multiplier applied to P50 when p10_instant has no entry + band_ratio: optional dict of timestamp string to (p10_ratio, p90_ratio), each relative to P50 + p10_fallback: multiplier applied to P50 when band_ratio has no entry + p90_fallback: multiplier applied to P50 when band_ratio is given but has no entry Returns: - dict of tz-aware UTC hour-start datetime to {"pv_estimate": kWh, "pv_estimate10": kWh} + dict of tz-aware UTC hour-start datetime to {"pv_estimate": kWh, "pv_estimate10": kWh}, + plus "pv_estimate90": kWh when band_ratio is given """ instant_kw = {} instant_stamps = [] @@ -102,15 +105,16 @@ def gti_hourly_to_period_kwh(times, gti_values, temp_values, wind_values, kwp, s wind = wind_values[idx] if idx < len(wind_values) and wind_values[idx] is not None else _DEFAULT_WIND_MS eta_temp = _temperature_efficiency(gti, temp, wind) pv50_inst = dp4((gti / 1000.0) * kwp * eta_temp * (1.0 - system_loss)) - raw_p10 = p10_instant.get(ts) if p10_instant else None - # p10_instant was computed without temperature derating; apply eta_temp now - pv10_inst = dp4(min(raw_p10 * eta_temp, pv50_inst) if raw_p10 is not None else pv50_inst * p10_fallback) + # The ratios scale the already derated P50, held to the side of it they belong on + p10_ratio, p90_ratio = band_ratio.get(ts, (p10_fallback, p90_fallback)) if band_ratio else (p10_fallback, p90_fallback) + pv10_inst = dp4(pv50_inst * min(max(p10_ratio, 0.0), 1.0)) + pv90_inst = dp4(pv50_inst * max(p90_ratio, 1.0)) try: stamp = datetime.strptime(ts, "%Y-%m-%dT%H:%M") stamp = stamp.replace(tzinfo=pytz.utc) except (ValueError, TypeError): continue - instant_kw[stamp] = (pv50_inst, pv10_inst) + instant_kw[stamp] = (pv50_inst, pv10_inst, pv90_inst) instant_stamps.append(stamp) period_data = {} @@ -119,16 +123,20 @@ def gti_hourly_to_period_kwh(times, gti_values, temp_values, wind_values, kwp, s next_stamp = instant_stamps[i + 1] if (next_stamp - stamp) != timedelta(hours=1): continue - pv50_start, pv10_start = instant_kw[stamp] - pv50_end, pv10_end = instant_kw[next_stamp] + pv50_start, pv10_start, pv90_start = instant_kw[stamp] + pv50_end, pv10_end, pv90_end = instant_kw[next_stamp] pv50 = dp4(0.5 * (pv50_start + pv50_end)) pv10 = dp4(0.5 * (pv10_start + pv10_end)) + pv90 = dp4(0.5 * (pv90_start + pv90_end)) if shading_factors and len(shading_factors) == 12: shading_month = shading_factors[stamp.month - 1] pv50 = dp4(pv50 * shading_month) pv10 = dp4(pv10 * shading_month) + pv90 = dp4(pv90 * shading_month) period_data[stamp] = {"pv_estimate": pv50, "pv_estimate10": pv10} + if band_ratio is not None: + period_data[stamp]["pv_estimate90"] = pv90 return period_data diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index 4df4eaec7..19f583c50 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -35,6 +35,9 @@ PV_CALIBRATION_LOWEST = 0.20 PV_CALIBRATION_HIGHEST = 4.0 +# Ceiling on the Open-Meteo ensemble's P90:median ratio. Near dawn and dusk the median is a few +# W/m2 and the ratio is unbounded; this is the same limit best_day_scaling is held to. +OPEN_METEO_P90_RATIO_MAX = 2.0 # apps.yaml args pointing at solar forecast entities an EXTERNAL integration publishes (typically # the HACS "Solcast PV Forecast" integration, when fetch_pv_forecast() falls back to reading HA @@ -94,6 +97,8 @@ def initialize( self.pv_scaling = pv_scaling self.open_meteo_forecast = open_meteo_forecast self.open_meteo_forecast_max_age = open_meteo_forecast_max_age + # Set by download_open_meteo_data(): True when the last download carried a real ensemble spread + self.open_meteo_ensemble_band = False self.solcast_requests_total = 0 self.solcast_failures_total = 0 self.forecast_solar_requests_total = 0 @@ -267,10 +272,17 @@ async def cache_get_url(self, url, params, max_age=8 * 60): URL_PERSONAL = "https://api.forecast.solar/{api_key}/estimate/{lat}/{lon}/{dec}/{az}/{kwp}?time=utc" URL_PERSONAL_DUAL = "https://api.forecast.solar/{api_key}/estimate/{lat}/{lon}/{dec1}/{az1}/{kwp1}/{dec2}/{az2}/{kwp2}?time=utc" - async def download_open_meteo_ensemble_data(self, lat, lon, tilt, az, kwp, system_loss): + async def download_open_meteo_ensemble_data(self, lat, lon, tilt, az): """ - Download Open-Meteo ensemble data for P10 solar estimate. - Returns a dict mapping ISO timestamp strings to P10 kW values. + Download Open-Meteo ensemble data for the spread of the solar estimate. + Returns a dict mapping ISO timestamp strings to (p10_ratio, p90_ratio): the ensemble's 10th + and 90th percentile irradiance for that hour relative to its own median. + + The ratios are applied to the deterministic P50 rather than the ensemble's absolute values + being used. The two are different model runs and can disagree on the level of a whole day, + so an absolute ensemble P10 can land above the deterministic P50 (leaving no downside at + all once clamped) or far below it; the ensemble's spread about its own centre is the part + that carries over. Hours where the ensemble median is zero have no ratio and are left out. """ url = "https://ensemble-api.open-meteo.com/v1/ensemble?models=icon_seamless&latitude={lat}&longitude={lon}&hourly=global_tilted_irradiance&tilt={tilt}&azimuth={az}&forecast_days=4&timezone=UTC".format(lat=lat, lon=lon, tilt=tilt, az=az) data = await self.cache_get_url(url, params={}, max_age=self.open_meteo_forecast_max_age * 60) @@ -291,23 +303,29 @@ async def download_open_meteo_ensemble_data(self, lat, lon, tilt, az, kwp, syste if val is not None: values.append(val) if not values: - result[ts] = 0.0 continue values.sort() - p10_idx = max(0, math.ceil(len(values) * 0.10) - 1) - gti_p10 = values[p10_idx] - result[ts] = dp4((gti_p10 / 1000.0) * kwp * (1.0 - system_loss)) + gti_p10 = values[max(0, math.ceil(len(values) * 0.10) - 1)] + gti_p50 = values[max(0, math.ceil(len(values) * 0.50) - 1)] + gti_p90 = values[max(0, math.ceil(len(values) * 0.90) - 1)] + if gti_p50 <= 0: + continue + result[ts] = (dp4(max(gti_p10, 0) / gti_p50), dp4(min(gti_p90 / gti_p50, OPEN_METEO_P90_RATIO_MAX))) return result async def download_open_meteo_data(self, configs=None): """ Download Open-Meteo forecast data and convert to PV power estimates. Uses GTI (global tilted irradiance) with simple temperature derating for P50, - and ensemble members for P10. Returns (sorted_data, max_kwh). + and the spread of the ensemble members for P10 and P90. Returns (sorted_data, max_kwh). If configs is provided it is used directly; otherwise self.open_meteo_forecast is used. """ period_data = {} max_kwh = 0 + # Whether every array that returned a forecast also got a real ensemble spread. Without one + # gti_hourly_to_period_kwh() fills pv_estimate10/90 with fixed fractions of P50, which + # fetch_pv_forecast() must not mistake for provider figures worth keeping. + ensemble_band_ok = True if configs is None: configs = self.open_meteo_forecast @@ -362,7 +380,10 @@ async def download_open_meteo_data(self, configs=None): self.log("Warn: SolarAPI: Open-Meteo data for lat {} lon {} has no hourly data".format(lat, lon)) continue - ensemble_p10 = await self.download_open_meteo_ensemble_data(lat, lon, tilt, az, kwp, system_loss) + ensemble_band = await self.download_open_meteo_ensemble_data(lat, lon, tilt, az) + if not ensemble_band: + self.log("Warn: SolarAPI: Open-Meteo ensemble data for lat {} lon {} could not be downloaded, the 10% and 90% scenarios will be created from history instead".format(lat, lon)) + ensemble_band_ok = False array_periods = gti_hourly_to_period_kwh( times, @@ -372,22 +393,25 @@ async def download_open_meteo_data(self, configs=None): kwp=kwp, system_loss=system_loss, shading_factors=shading_factors, - p10_instant=ensemble_p10, + band_ratio=ensemble_band, ) for stamp, values in array_periods.items(): pv50 = values["pv_estimate"] pv10 = values["pv_estimate10"] + pv90 = values["pv_estimate90"] if stamp in period_data: period_data[stamp]["pv_estimate"] = dp4(period_data[stamp]["pv_estimate"] + pv50) period_data[stamp]["pv_estimate10"] = dp4(period_data[stamp]["pv_estimate10"] + pv10) + period_data[stamp]["pv_estimate90"] = dp4(period_data[stamp]["pv_estimate90"] + pv90) else: - period_data[stamp] = {"period_start": stamp.strftime(TIME_FORMAT), "pv_estimate": pv50, "pv_estimate10": pv10} + period_data[stamp] = {"period_start": stamp.strftime(TIME_FORMAT), "pv_estimate": pv50, "pv_estimate10": pv10, "pv_estimate90": pv90} sorted_data = [] if period_data: for key in sorted(period_data.keys()): sorted_data.append(period_data[key]) + self.open_meteo_ensemble_band = ensemble_band_ok and bool(sorted_data) self.log("SolarAPI: Open-Meteo returned {} data points".format(len(sorted_data))) return sorted_data, max_kwh @@ -1142,11 +1166,14 @@ def pv_forecast_history(self, days): self.log("Warn: SolarAPI: PV Calibration: could not read any forecast history from sensor.{}_pv_forecast_h0_uncalibrated or sensor.{}_pv_forecast_h0 - calibration will be disabled until history builds up".format(self.prefix, self.prefix)) return history - def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data, create_pv10, divide_by, max_kwh, forecast_days, period=None): + def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data, create_pv10, divide_by, max_kwh, forecast_days, period=None, calibrate_band=False): """ Perform PV calibration based on historical data and forecast data. This will adjust the forecast data based on historical PV production and forecast data. It will also create pv_estimate10 and pv_estimate90 data if create_pv10 is True. + calibrate_band is for a provider that supplies its own pv_estimate10/pv_estimate90 (so + create_pv10 is False): they are kept, and moved with P50 by whatever calibration and array + ceiling were applied to it, so they keep the provider's ratio to the P50 that is returned. """ # If no period is given, default to the plan interval (backward-compatible for unit tests). if period is None: @@ -1419,6 +1446,12 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_m # Per-slot best_day_scaling for the planner's p90 series, held back so the create_pv10 block # below can apply the same ceiling this loop applies to the published pv_estimate90. slot_best_scaling = {} + # The user's switch, as opposed to enabled_calibration above which is "enough history to + # calibrate from". It picks the series returned as P50, and so the one the band is built on. + calibration_on = self.get_arg("metric_pv_calibration_enable", default=True) + # The raw forecast as the planner gets it with calibration off: unscaled, but still held to + # the array ceiling in the loop below. A copy, so the caller's series is left alone. + pv_forecast_minute_raw = dict(pv_forecast_minute) for minute in range(0, max(pv_forecast_minute.keys()) + 1, self.plan_interval_minutes): pv_value = 0 raw_value = 0 @@ -1447,17 +1480,27 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_m # for. Only the array limit may cap the upside case. capped_p50 = min(pv_value, capped_data) pv_estimateCL[minute] = dp4(capped_p50) - pv_estimate10[minute] = dp4(capped_p50 * worst_day_scaling) - pv_estimate90[minute] = dp4(min(capped_p50 * best_day_scaling, ceiling_slot)) - # The planner's p90 series (built in the create_pv10 block below from the capped + # The 10%/90% band is a scaling of whichever series is handed to the planner as P50: + # the capped calibrated one, or the raw forecast when the user has switched calibration + # off. Scaling the calibrated series regardless left the band and its centre line on two + # different curves, so P10 could sit above P50 and P90 below it (GH#5345). + # + # Switching calibration off turns off the scaling, not the array ceiling: a raw forecast + # above what the array can produce is clipped to ceiling_slot either way (the warning + # below says as much), so P50 never sits above the ceiling P90 is clamped to. + band_p50 = capped_p50 if calibration_on else min(raw_value, ceiling_slot) + pv_estimate10[minute] = dp4(band_p50 * worst_day_scaling) + pv_estimate90[minute] = dp4(min(band_p50 * best_day_scaling, ceiling_slot)) + + # The planner's p90 series (built in the create_pv10 block below from the same # per-minute data) must land on the same ceiling as pv_estimate90 above, or the two # disagree exactly where the comment above says they must agree. ceiling_slot is kWh per # plan interval, so the clamp cannot be applied per minute; record the scaling that - # holds this slot's p90 total at min(capped_p50 * best_day_scaling, ceiling_slot) instead + # holds this slot's p90 total at min(band_p50 * best_day_scaling, ceiling_slot) instead # and let the block below scale every minute of the slot by it. An empty slot has no # ratio to take, and needs no clamp either - scaling zero by anything stays zero. - slot_best_scaling[minute] = min(best_day_scaling, ceiling_slot / capped_p50) if capped_p50 > 0 else best_day_scaling + slot_best_scaling[minute] = min(best_day_scaling, ceiling_slot / band_p50) if band_p50 > 0 else best_day_scaling # Apply the same cap to the per-minute data the planner consumes. Scale rather than # clamp per minute: capped_data is kWh per plan interval, not per minute. @@ -1468,6 +1511,13 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_m pv_forecast_minute_adjusted[minute + offset] = dp4(pv_forecast_minute_adjusted[minute + offset] * scale_down) capped_slots += 1 + # And the ceiling alone to the raw series used when calibration is switched off. + if raw_value > ceiling_slot: + scale_down = ceiling_slot / raw_value + for offset in range(0, self.plan_interval_minutes, 1): + if (minute + offset) in pv_forecast_minute_raw: + pv_forecast_minute_raw[minute + offset] = dp4(pv_forecast_minute_raw[minute + offset] * scale_down) + if capped_slots: ceiling_kw = ceiling_slot * 60 / self.plan_interval_minutes self.log("SolarAPI: PV Calibration: Capped {} slots to the array ceiling ({}kW observed peak, {}kW ceiling)".format(capped_slots, dp2(max_pv_power_hist), dp2(ceiling_kw))) @@ -1487,6 +1537,33 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_m ) ) + # Do we use calibrated or raw data? The band below is built from whichever is returned. + pv_forecast_minute_used = pv_forecast_minute_adjusted if calibration_on else pv_forecast_minute_raw + + # A provider's own band (Open-Meteo's ensemble spread) says how far either side of P50 that + # particular hour could land, which the flat worst/best day scaling cannot. Keep it, but + # move it with the P50 it is paired with: calibration says the array produces some multiple + # of the forecast in this slot, and that holds for the pessimistic and optimistic scenarios + # as much as the central one. Scaling each minute by used / raw preserves the provider's + # ratios to P50 and picks up the array cap applied above (the only thing that moves the raw + # series when calibration is off). P10 is held at or below P50; P90 at or above it and, as + # for a created P90, no higher than the array ceiling unless P50 itself is. + if calibrate_band and not create_pv10: + pv_estimate10 = {} + pv_estimate90 = {} + ceiling_minute = ceiling_slot / self.plan_interval_minutes + for minute in range(0, max(pv_forecast_minute.keys()) + 1): + raw_minute = pv_forecast_minute.get(minute, 0) + used_minute = pv_forecast_minute_used.get(minute, 0) + scale = used_minute / raw_minute if raw_minute > 0 else 1.0 + pv10_minute = dp4(min(pv_forecast_minute10.get(minute, 0) * scale, used_minute)) + pv90_minute = dp4(max(min(pv_forecast_minute90.get(minute, 0) * scale, ceiling_minute), used_minute)) + pv_forecast_minute10[minute] = pv10_minute + pv_forecast_minute90[minute] = pv90_minute + slot_start = int(minute / self.plan_interval_minutes) * self.plan_interval_minutes + pv_estimate10[slot_start] = pv_estimate10.get(slot_start, 0) + pv10_minute + pv_estimate90[slot_start] = pv_estimate90.get(slot_start, 0) + pv90_minute + for entry in pv_forecast_data: period_start = entry.get("period_start", "") if period_start: @@ -1521,16 +1598,16 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_m # When we store the data we have to reverse the divide_by factor if has_calibrated: entry["pv_estimateCL"] = calibrated * divide_by - if create_pv10 and has_calibrated10: + if (create_pv10 or calibrate_band) and has_calibrated10: entry["pv_estimate10"] = calibrated10 * divide_by - if create_pv10 and has_calibrated90: + if (create_pv10 or calibrate_band) and has_calibrated90: entry["pv_estimate90"] = calibrated90 * divide_by # Creation of PV10 data using worst day scaling factor if create_pv10: capped_best_slots = 0 - for minute in range(0, max(pv_forecast_minute_adjusted.keys()) + 1): - pv_value = pv_forecast_minute_adjusted.get(minute, 0) + for minute in range(0, max(pv_forecast_minute_used.keys()) + 1): + pv_value = pv_forecast_minute_used.get(minute, 0) # Use the worst day scaling factor to create pv_estimate10. No ceiling clamp is # needed: worst_day_scaling is capped at 1.0 above, so this can only scale down. pv_forecast_minute10[minute] = dp4(pv_value * worst_day_scaling) @@ -1550,12 +1627,9 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_m ) ) - # Do we use calibrated or raw data? - if self.get_arg("metric_pv_calibration_enable", default=True): + if calibration_on: self.log("SolarAPI: PV Calibration: Using calibrated PV data") - return pv_forecast_minute_adjusted, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data - else: - return pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data + return pv_forecast_minute_used, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data def pack_and_store_forecast(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90): """ @@ -1808,9 +1882,18 @@ async def fetch_pv_forecast(self): # of double-counting a site configured against two providers. self.base.resolve_pv_array_kwp(max_kwh) + # Open-Meteo's ensemble supplies a real spread, so keep the P10 and P90 built from it + # (calibrated alongside P50) rather than replacing them with ones created from history. + # active_source, not configured_source: a Forecast.Solar fallback or primary that + # actually served the data has no band of its own. + calibrate_band = False + if active_source == "open_meteo" and self.open_meteo_ensemble_band: + create_pv10 = False + calibrate_band = True + # Run calibration on the data pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data = self.pv_calibration( - pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data, create_pv10, divide_by / period, max_kwh, self.forecast_days, period + pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data, create_pv10, divide_by / period, max_kwh, self.forecast_days, period, calibrate_band=calibrate_band ) self.publish_pv_stats(pv_forecast_data, divide_by / period, period) self.pack_and_store_forecast(pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90) diff --git a/apps/predbat/tests/test_open_meteo.py b/apps/predbat/tests/test_open_meteo.py index fb9bec7c6..f65d8b24f 100644 --- a/apps/predbat/tests/test_open_meteo.py +++ b/apps/predbat/tests/test_open_meteo.py @@ -56,37 +56,50 @@ def _make_ensemble_response(times=None, members=None): # ============================================================================ -def test_ensemble_returns_p10_values(my_predbat): +def test_ensemble_returns_band_ratios(my_predbat): """ - download_open_meteo_ensemble_data should return a dict of ts→kW10 - where each value is the 10th-percentile GTI across members, converted to kW. + download_open_meteo_ensemble_data should return a dict of ts -> (p10_ratio, p90_ratio), + the 10th and 90th percentile GTI across members relative to the ensemble's own median. + Hours with a zero median have no ratio and are left out, and the P90 ratio is capped. """ - print(" - test_ensemble_returns_p10_values") + print(" - test_ensemble_returns_band_ratios") failed = False test_api = create_test_solar_api() try: test_api.solar.open_meteo_forecast_max_age = 1.0 - ensemble_data = _make_ensemble_response() + ensemble_data = _make_ensemble_response( + times=["2025-06-15T04:00", "2025-06-15T05:00", "2025-06-15T12:00"], + members={ + "global_tilted_irradiance_member01": [0.0, 1.0, 400.0], + "global_tilted_irradiance_member02": [0.0, 2.0, 450.0], + "global_tilted_irradiance_member03": [5.0, 40.0, 480.0], + }, + ) test_api.set_mock_response("ensemble-api.open-meteo.com", ensemble_data) def create_mock_session(*args, **kwargs): return test_api.mock_aiohttp_session() - kwp = 3.0 - system_loss = 0.0 # simplify: 0% loss so kW = GTI_kWm2 * kwp with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): - result = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0, kwp, system_loss)) + result = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0)) - # For 3 members at 2025-06-15T12:00: [400, 450, 480] sorted - # p10_idx = max(0, int(3 * 0.1) - 1) = 0 -> gti_p10 = 400 - # kW = (400 / 1000) * 3.0 * (1 - 0.0) = 1.2 - expected_12 = round((400.0 / 1000.0) * kwp * (1.0 - system_loss), 4) + # Three members sorted: P10 is the lowest, the median the middle and P90 the highest. + # 12:00 -> [400, 450, 480]: 400/450 and 480/450 + expected_12 = (round(400.0 / 450.0, 4), round(480.0 / 450.0, 4)) if "2025-06-15T12:00" not in result: print("ERROR: Expected key '2025-06-15T12:00' in ensemble result") failed = True - elif abs(result["2025-06-15T12:00"] - expected_12) > 0.001: - print(f"ERROR: ensemble p10 at 12:00: expected {expected_12}, got {result['2025-06-15T12:00']}") + elif abs(result["2025-06-15T12:00"][0] - expected_12[0]) > 0.001 or abs(result["2025-06-15T12:00"][1] - expected_12[1]) > 0.001: + print(f"ERROR: ensemble ratios at 12:00: expected {expected_12}, got {result['2025-06-15T12:00']}") + failed = True + # 04:00 -> [0, 0, 5]: the median is zero, so there is no ratio to take + if "2025-06-15T04:00" in result: + print(f"ERROR: an hour with a zero ensemble median should be left out, got {result['2025-06-15T04:00']}") + failed = True + # 05:00 -> [1, 2, 40]: 40/2 = 20x is capped at 2.0 + if result.get("2025-06-15T05:00") != (0.5, 2.0): + print(f"ERROR: ensemble ratios at 05:00: expected (0.5, 2.0) with the P90 ratio capped, got {result.get('2025-06-15T05:00')}") failed = True finally: test_api.cleanup() @@ -112,7 +125,7 @@ def create_mock_session(*args, **kwargs): return test_api.mock_aiohttp_session() with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): - result = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0, 3.0, 0.14)) + result = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0)) if result != {}: print(f"ERROR: Expected empty dict, got {result}") @@ -139,7 +152,7 @@ def create_mock_session(*args, **kwargs): return test_api.mock_aiohttp_session() with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): - result = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0, 3.0, 0.14)) + result = run_async(test_api.solar.download_open_meteo_ensemble_data(51.5, -0.1, 35, 0)) if result != {}: print(f"ERROR: Expected empty dict on HTTP failure, got {result}") @@ -746,7 +759,7 @@ def run_open_meteo_tests(my_predbat): print("Running Open-Meteo solar forecast tests...") failed = False - failed |= test_ensemble_returns_p10_values(my_predbat) + failed |= test_ensemble_returns_band_ratios(my_predbat) failed |= test_ensemble_empty_on_no_members(my_predbat) failed |= test_ensemble_empty_on_http_failure(my_predbat) failed |= test_download_open_meteo_data_basic(my_predbat) diff --git a/apps/predbat/tests/test_solar_model.py b/apps/predbat/tests/test_solar_model.py index d597827af..189edcbdb 100644 --- a/apps/predbat/tests/test_solar_model.py +++ b/apps/predbat/tests/test_solar_model.py @@ -101,18 +101,26 @@ def test_solar_model(my_predbat): print(" ERROR: p10_fallback 0.5 not applied, got {}".format(half_first["pv_estimate10"])) failed = True - print("Test: p10_instant overrides the fallback and is capped at P50") - ensemble = {FLAT_TIMES[index]: 0.1 for index in range(4)} - with_ensemble = gti_hourly_to_period_kwh(FLAT_TIMES, flat_gti, flat_temp, flat_wind, kwp=1.0, system_loss=0.0, p10_instant=ensemble) + print("Test: band_ratio scales P50 into P10 and P90, each held to its own side of P50") + if "pv_estimate90" in first: + print(" ERROR: pv_estimate90 should only be returned when band_ratio is given") + failed = True + ensemble = {FLAT_TIMES[index]: (0.4, 1.6) for index in range(4)} + with_ensemble = gti_hourly_to_period_kwh(FLAT_TIMES, flat_gti, flat_temp, flat_wind, kwp=1.0, system_loss=0.0, band_ratio=ensemble) ensemble_first = with_ensemble[stamp_for(FLAT_TIMES[0])] - if ensemble_first["pv_estimate10"] >= ensemble_first["pv_estimate"]: - print(" ERROR: an ensemble P10 below P50 should stay below it, got {}".format(ensemble_first["pv_estimate10"])) + if abs(ensemble_first["pv_estimate10"] - ensemble_first["pv_estimate"] * 0.4) > 0.0002 or abs(ensemble_first["pv_estimate90"] - ensemble_first["pv_estimate"] * 1.6) > 0.0002: + print(" ERROR: band ratios 0.4 / 1.6 not applied to P50 {}, got {} / {}".format(ensemble_first["pv_estimate"], ensemble_first["pv_estimate10"], ensemble_first["pv_estimate90"])) failed = True - huge = {FLAT_TIMES[index]: 99.0 for index in range(4)} - capped = gti_hourly_to_period_kwh(FLAT_TIMES, flat_gti, flat_temp, flat_wind, kwp=1.0, system_loss=0.0, p10_instant=huge) + inverted = {FLAT_TIMES[index]: (1.5, 0.5) for index in range(4)} + capped = gti_hourly_to_period_kwh(FLAT_TIMES, flat_gti, flat_temp, flat_wind, kwp=1.0, system_loss=0.0, band_ratio=inverted) capped_first = capped[stamp_for(FLAT_TIMES[0])] - if abs(capped_first["pv_estimate10"] - capped_first["pv_estimate"]) > 0.0001: - print(" ERROR: an ensemble P10 above P50 should be capped at P50, got {}".format(capped_first["pv_estimate10"])) + if abs(capped_first["pv_estimate10"] - capped_first["pv_estimate"]) > 0.0001 or abs(capped_first["pv_estimate90"] - capped_first["pv_estimate"]) > 0.0001: + print(" ERROR: a P10 ratio above 1 and a P90 ratio below 1 should both be held at P50, got {} / {}".format(capped_first["pv_estimate10"], capped_first["pv_estimate90"])) + failed = True + sparse = gti_hourly_to_period_kwh(FLAT_TIMES, flat_gti, flat_temp, flat_wind, kwp=1.0, system_loss=0.0, band_ratio={}) + sparse_first = sparse[stamp_for(FLAT_TIMES[0])] + if abs(sparse_first["pv_estimate10"] - sparse_first["pv_estimate"] * 0.7) > 0.0002 or abs(sparse_first["pv_estimate90"] - sparse_first["pv_estimate"] * 1.3) > 0.0002: + print(" ERROR: hours missing from band_ratio should fall back to 0.7 / 1.3, got {} / {}".format(sparse_first["pv_estimate10"], sparse_first["pv_estimate90"])) failed = True print("Test: shading_factors apply the correct month") diff --git a/apps/predbat/tests/test_solcast.py b/apps/predbat/tests/test_solcast.py index 851c7b55e..cf545424e 100644 --- a/apps/predbat/tests/test_solcast.py +++ b/apps/predbat/tests/test_solcast.py @@ -4383,6 +4383,355 @@ def test_pv_calibration_cap_published_pv10_matches_planner(my_predbat): return failed +def _band_scenario(calibration_on, raw_kw=1.0, max_kwh=20.0, band_ratios=None): + """Run pv_calibration with a history that calibrates the morning up and the afternoon down. + + Three past days each have two one-hour generation windows against a recorded forecast of + 1 kW: the morning window (10:00) produced 1.5x the forecast and the afternoon window (13:00) + 0.4x, so slot_adjustment is well above 1.0 in one and well below it in the other. The days + themselves ran at 0.8x, 1.0x and 1.2x of that shape, which gives a worst/best day scaling + either side of 1.0 without changing the shape. Today's raw forecast is raw_kw in both windows; + the default 1 kW sits below the 1.8 kW observed peak, so the array cap leaves room for the + morning to be scaled up. + + band_ratios, when given, maps each window start to the provider's own (P10, P90) as fractions + of its P50. The forecast then carries that band (as Open-Meteo's ensemble spread does) and + pv_calibration is asked to keep it rather than create one. + + Returns (test_api, raw_m, adj_m, adj_m10, adj_m90, adj_data, windows). The caller owns the + returned test_api and must call cleanup() on it. + """ + windows = {600: 1.5, 780: 0.4} + day_factors = [0.8, 1.0, 1.2] + window_length = 60 + test_api = create_test_solar_api() + solar = test_api.solar + base = test_api.mock_base + base.set_arg("metric_pv_calibration_enable", calibration_on) + plan_interval = base.plan_interval_minutes + minutes_now = base.minutes_now + days_back = len(day_factors) + + # Cumulative pv_today kWh keyed by minutes-ago + hist = {} + for day_idx, day_factor in enumerate(day_factors): + midnight_ago = (day_idx + 1) * 1440 + minutes_now + for step in range(0, 24 * 60, 5): + minute_ago = midnight_ago - step + if minute_ago < 0: + continue + cumulative = 0.0 + for start, shape in windows.items(): + cumulative += shape * day_factor * min(max(step - start, 0), window_length) / 60.0 + hist[minute_ago] = cumulative + + # Recorded forecast history of 1 kW in both windows on every past day + pv_forecast_hist = {} + for day_num in range(1, days_back + 1): + for start in windows: + for m_of_day in range(start, start + window_length): + pv_forecast_hist[day_num * 1440 + (minutes_now - m_of_day)] = 1.0 + + def mock_minute_import_export(max_days_prev, now_utc, key, scale=1.0, required_unit=None, increment=True, smoothing=True, pad=True, _hist=hist): + """Return the synthetic pv_today history.""" + return dict(_hist) if key == "pv_today" else {} + + base.minute_data_import_export = mock_minute_import_export + solar.get_history_wrapper = lambda entity_id, days, required=False: [] + + def in_window(minute): + """True when the minute of today falls inside a generation window.""" + return any(start <= minute < start + window_length for start in windows) + + raw_m = {m: (raw_kw / 60.0) if in_window(m) else 0.0 for m in range(4 * 24 * 60)} + raw_m10 = {} + raw_m90 = {} + if band_ratios: + for m in raw_m: + p10_ratio, p90_ratio = next((band_ratios[start] for start in windows if start <= m < start + window_length), (0.0, 0.0)) + raw_m10[m] = raw_m[m] * p10_ratio + raw_m90[m] = raw_m[m] * p90_ratio + + midnight = datetime(2025, 6, 15, 0, 0, 0, tzinfo=pytz.utc) + pv_data = [] + for start in windows: + for slot in range(start, start + window_length, plan_interval): + ts = midnight + timedelta(minutes=slot) + entry = {"period_start": ts.strftime("%Y-%m-%dT%H:%M:%S+0000"), "pv_estimate": raw_kw * plan_interval / 60.0} + if band_ratios: + entry["pv_estimate10"] = entry["pv_estimate"] * band_ratios[start][0] + entry["pv_estimate90"] = entry["pv_estimate"] * band_ratios[start][1] + pv_data.append(entry) + + with patch("solcast.history_attribute_to_minute_data", return_value=(pv_forecast_hist, days_back)): + if band_ratios: + adj_m, adj_m10, adj_m90, adj_data = solar.pv_calibration(dict(raw_m), raw_m10, raw_m90, pv_data, create_pv10=False, divide_by=1.0, max_kwh=max_kwh, forecast_days=solar.forecast_days, calibrate_band=True) + else: + adj_m, adj_m10, adj_m90, adj_data = solar.pv_calibration(dict(raw_m), {}, {}, pv_data, create_pv10=True, divide_by=1.0, max_kwh=max_kwh, forecast_days=solar.forecast_days) + + return test_api, raw_m, adj_m, adj_m10, adj_m90, adj_data, windows + + +def test_pv_calibration_off_band_built_from_raw(my_predbat): + """ + With the calibration switch off, the 10% and 90% scenarios must be built from the raw forecast. + + Regression test for GH#5345: pv_calibration always built pv_forecast_minute10/90 (and the + published pv_estimate10/90) from the calibrated series, and only consulted + metric_pv_calibration_enable at the final return, where it swapped P50 back to the raw + series. The centre line and its band then came from two different curves, so wherever + calibration scaled a slot up by more than 1 / worst_day_scaling the "pessimistic" P10 sat + above P50, and wherever it scaled a slot down by more than 1 / best_day_scaling the + "optimistic" P90 sat below it. That also made the P50-P10 gap the cloud model reads arbitrary. + + _band_scenario calibrates the morning up (1.5x) and the afternoon down (0.4x), with a + worst/best day scaling of roughly 0.8/1.2, so before the fix both inversions appear. + """ + print(" - test_pv_calibration_off_band_built_from_raw") + failed = False + tolerance = 0.0002 # the series are rounded to 4 decimal places + + test_api, raw_m, adj_m, adj_m10, adj_m90, adj_data, windows = _band_scenario(calibration_on=False) + try: + worst = test_api.solar.pv_calibration_worst_scaling + best = test_api.solar.pv_calibration_best_scaling + if not (worst < 0.95 and best > 1.05): + print("ERROR: scenario did not produce a band, worst {} best {}".format(worst, best)) + failed = True + + for start, shape in windows.items(): + minute = start + 30 + raw = raw_m[minute] + if abs(adj_m[minute] - raw) > tolerance: + print("ERROR: with calibration off P50 at minute {} is {}, expected the raw {}".format(minute, adj_m[minute], raw)) + failed = True + if abs(adj_m10[minute] - raw * worst) > tolerance: + print("ERROR: with calibration off P10 at minute {} is {}, expected raw {} x worst {} = {} (window calibrated {}x)".format(minute, adj_m10[minute], raw, worst, raw * worst, shape)) + failed = True + if abs(adj_m90[minute] - raw * best) > tolerance: + print("ERROR: with calibration off P90 at minute {} is {}, expected raw {} x best {} = {} (window calibrated {}x)".format(minute, adj_m90[minute], raw, best, raw * best, shape)) + failed = True + + for minute in sorted(raw_m): + if not (adj_m10.get(minute, 0) <= adj_m[minute] + tolerance and adj_m[minute] <= adj_m90.get(minute, 0) + tolerance): + print("ERROR: with calibration off minute {} breaks P10 <= P50 <= P90: {} / {} / {}".format(minute, adj_m10.get(minute, 0), adj_m[minute], adj_m90.get(minute, 0))) + failed = True + break + + calibration_moved = False + for entry in adj_data: + p50 = entry["pv_estimate"] + p10 = entry.get("pv_estimate10") + p90 = entry.get("pv_estimate90") + if p10 is None or p90 is None: + print("ERROR: published entry {} is missing pv_estimate10/pv_estimate90".format(entry["period_start"])) + failed = True + break + if not (p10 <= p50 + tolerance and p50 <= p90 + tolerance): + print("ERROR: with calibration off published entry {} breaks P10 <= P50 <= P90: {} / {} / {}".format(entry["period_start"], p10, p50, p90)) + failed = True + break + if abs(p10 - p50 * worst) > tolerance or abs(p90 - p50 * best) > tolerance: + print("ERROR: with calibration off published entry {} band {} / {} is not the raw {} scaled by {} / {}".format(entry["period_start"], p10, p90, p50, worst, best)) + failed = True + break + if abs(entry.get("pv_estimateCL", p50) - p50) > 0.1 * p50: + calibration_moved = True + if not calibration_moved: + print("ERROR: pv_estimateCL never differs from pv_estimate - the scenario is not calibrating, so the test proves nothing") + failed = True + finally: + test_api.cleanup() + + # Raw forecast above the array ceiling (10 kW against 1.2 x 4 kW = 4.8 kW): switching calibration + # off does not switch the ceiling off, so P50 is clipped to it and P90 sits on it, not below P50. + test_api, raw_m, adj_m, adj_m10, adj_m90, adj_data, windows = _band_scenario(calibration_on=False, raw_kw=10.0, max_kwh=4.0) + try: + ceiling_minute = 4.8 / 60.0 + worst = test_api.solar.pv_calibration_worst_scaling + for start in windows: + minute = start + 30 + if abs(adj_m[minute] - ceiling_minute) > tolerance: + print("ERROR: with calibration off and a raw forecast above the ceiling, P50 {} at minute {} is not clipped to the ceiling {}".format(adj_m[minute], minute, ceiling_minute)) + failed = True + if abs(adj_m90[minute] - ceiling_minute) > tolerance: + print("ERROR: with calibration off and a raw forecast above the ceiling, P90 {} at minute {} is not held at the ceiling {}".format(adj_m90[minute], minute, ceiling_minute)) + failed = True + if abs(adj_m10[minute] - ceiling_minute * worst) > tolerance: + print("ERROR: with calibration off and a raw forecast above the ceiling, P10 {} at minute {} is not the clipped P50 x worst {}".format(adj_m10[minute], minute, worst)) + failed = True + if abs(raw_m[630] - 10.0 / 60.0) > 1e-9: + print("ERROR: the caller's raw series was modified by the ceiling clip") + failed = True + finally: + test_api.cleanup() + + # Calibration on is unchanged: the band is a scaling of the calibrated series that is returned. + test_api, raw_m, adj_m, adj_m10, adj_m90, adj_data, windows = _band_scenario(calibration_on=True) + try: + worst = test_api.solar.pv_calibration_worst_scaling + best = test_api.solar.pv_calibration_best_scaling + for start in windows: + minute = start + 30 + if abs(adj_m[minute] - raw_m[minute]) < 0.1 * raw_m[minute]: + print("ERROR: with calibration on P50 at minute {} is {}, expected it to move away from the raw {}".format(minute, adj_m[minute], raw_m[minute])) + failed = True + if abs(adj_m10[minute] - adj_m[minute] * worst) > tolerance or abs(adj_m90[minute] - adj_m[minute] * best) > tolerance: + print("ERROR: with calibration on the band {} / {} at minute {} is not the calibrated {} scaled by {} / {}".format(adj_m10[minute], adj_m90[minute], minute, adj_m[minute], worst, best)) + failed = True + finally: + test_api.cleanup() + + return failed + + +def test_pv_calibration_keeps_provider_band(my_predbat): + """ + A provider's own P10 and P90 must survive calibration, following the P50 they are returned with. + + Open-Meteo's ensemble gives a spread that varies hour by hour, but pv_calibration used to be + told to create the band for it, which replaced the provider's figures with P50 scaled by the + flat worst/best day scaling. With calibrate_band the provider's P10 and P90 are kept and scaled + by exactly the calibration applied to P50 in that minute, so their ratios to P50 are preserved. + + The provider band here is 0.9x/1.1x of P50 in the morning and 0.3x/1.8x in the afternoon, + neither of which is the worst/best day scaling (about 0.81/1.22) a created band would show. + """ + print(" - test_pv_calibration_keeps_provider_band") + failed = False + tolerance = 0.0002 # the series are rounded to 4 decimal places + band_ratios = {600: (0.9, 1.1), 780: (0.3, 1.8)} + + for calibration_on in (True, False): + label = "on" if calibration_on else "off" + test_api, raw_m, adj_m, adj_m10, adj_m90, adj_data, windows = _band_scenario(calibration_on=calibration_on, band_ratios=band_ratios) + try: + plan_interval = test_api.mock_base.plan_interval_minutes + for start in windows: + minute = start + 30 + p10_ratio, p90_ratio = band_ratios[start] + moved = abs(adj_m[minute] - raw_m[minute]) > 0.1 * raw_m[minute] + if moved != calibration_on: + print("ERROR: calibration {}: P50 at minute {} is {} against the raw {}".format(label, minute, adj_m[minute], raw_m[minute])) + failed = True + if abs(adj_m10[minute] - adj_m[minute] * p10_ratio) > tolerance: + print("ERROR: calibration {}: P10 at minute {} is {}, expected the provider's {} x P50 {} = {}".format(label, minute, adj_m10[minute], p10_ratio, adj_m[minute], adj_m[minute] * p10_ratio)) + failed = True + if abs(adj_m90[minute] - adj_m[minute] * p90_ratio) > tolerance: + print("ERROR: calibration {}: P90 at minute {} is {}, expected the provider's {} x P50 {} = {}".format(label, minute, adj_m90[minute], p90_ratio, adj_m[minute], adj_m[minute] * p90_ratio)) + failed = True + + midnight = datetime(2025, 6, 15, 0, 0, 0, tzinfo=pytz.utc) + for entry in adj_data: + slot = int((datetime.strptime(entry["period_start"], "%Y-%m-%dT%H:%M:%S%z") - midnight).total_seconds() / 60) + for key, series in (("pv_estimate10", adj_m10), ("pv_estimate90", adj_m90)): + planner_total = sum(series.get(slot + offset, 0) for offset in range(plan_interval)) + if abs(entry[key] - planner_total) > tolerance * plan_interval: + print("ERROR: calibration {}: published {} {} at {} does not match the planner's total {}".format(label, key, entry[key], entry["period_start"], planner_total)) + failed = True + finally: + test_api.cleanup() + + # A provider P90 above the array ceiling (1 kW x 1.8 against 1.2 x 1 kW) is held at the ceiling, + # and a raw P50 above it (10 kW against 1.2 x 4 kW) drags the whole band down with it. + test_api, raw_m, adj_m, adj_m10, adj_m90, adj_data, windows = _band_scenario(calibration_on=False, max_kwh=1.0, band_ratios=band_ratios) + try: + ceiling_minute = max(1.2 * 1.0, 1.8) / 60.0 + if abs(adj_m90[810] - min(raw_m[810] * 1.8, ceiling_minute)) > tolerance or adj_m90[810] > ceiling_minute + tolerance: + print("ERROR: provider P90 {} at minute 810 is not held at the array ceiling {}".format(adj_m90[810], ceiling_minute)) + failed = True + finally: + test_api.cleanup() + test_api, raw_m, adj_m, adj_m10, adj_m90, adj_data, windows = _band_scenario(calibration_on=False, raw_kw=10.0, max_kwh=4.0, band_ratios=band_ratios) + try: + ceiling_minute = 4.8 / 60.0 + for start in windows: + minute = start + 30 + if abs(adj_m[minute] - ceiling_minute) > tolerance or abs(adj_m90[minute] - ceiling_minute) > tolerance or abs(adj_m10[minute] - ceiling_minute * band_ratios[start][0]) > tolerance: + print("ERROR: raw forecast above the ceiling: band {} / {} / {} at minute {} is not clipped to the ceiling {} with the provider's P10 ratio".format(adj_m10[minute], adj_m[minute], adj_m90[minute], minute, ceiling_minute)) + failed = True + finally: + test_api.cleanup() + + return failed + + +def test_fetch_pv_forecast_open_meteo_uses_ensemble_band(my_predbat): + """ + fetch_pv_forecast must keep Open-Meteo's ensemble band when it has one, and create one when not. + + The ensemble's P10 and P90 are taken as ratios of the ensemble's own median and applied to the + deterministic P50, because the two are different model runs and can disagree on the level of + the day. Here the three members sit at 0.5x, 1.0x and 1.5x of a median that is deliberately + twice the deterministic irradiance: the published band must still be 0.5x / 1.5x of the + deterministic P50, where the absolute ensemble P10 would have equalled P50 and left no gap. + + With the ensemble endpoint answering, pv_calibration is called with create_pv10 False and + calibrate_band True. When the ensemble download fails the entries only carry fixed 0.7x / 1.3x + placeholders, so the history-based band is the better figure and create_pv10 stays True. + """ + print(" - test_fetch_pv_forecast_open_meteo_uses_ensemble_band") + failed = False + + for with_ensemble in (True, False): + test_api = create_test_solar_api() + try: + test_api.solar.open_meteo_forecast = [{"latitude": 51.5, "longitude": -0.1, "declination": 30, "azimuth": 0, "kwp": 3.0}] + test_api.solar.open_meteo_forecast_max_age = 1.0 + times = ["2025-06-15T12:00", "2025-06-15T13:00", "2025-06-15T14:00"] + # "//api." so the forecast mock does not also match the ensemble-api host: the mock session + # returns the first response whose key is a substring of the URL. + test_api.set_mock_response("//api.open-meteo.com", {"hourly": {"time": times, "global_tilted_irradiance": [500.0, 500.0, 500.0], "temperature_2m": [25.0, 25.0, 25.0], "wind_speed_10m": [1.0, 1.0, 1.0]}}) + if with_ensemble: + members = {"global_tilted_irradiance_member01": [500.0] * 3, "global_tilted_irradiance_member02": [1000.0] * 3, "global_tilted_irradiance_member03": [1500.0] * 3} + test_api.set_mock_response("ensemble-api.open-meteo.com", {"hourly": dict(members, time=times)}) + else: + test_api.set_mock_response("ensemble-api.open-meteo.com", {}, 500) + + captured = {} + real_pv_calibration = test_api.solar.pv_calibration + + def spy_pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data, create_pv10, *args, _captured=captured, _real=real_pv_calibration, **kwargs): + """Record how pv_calibration was asked to treat the band, then run it.""" + _captured["create_pv10"] = create_pv10 + _captured["calibrate_band"] = kwargs.get("calibrate_band") + return _real(pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_data, create_pv10, *args, **kwargs) + + test_api.solar.pv_calibration = spy_pv_calibration + + def create_mock_session(*args, _api=test_api, **kwargs): + """Create a mock aiohttp session.""" + return _api.mock_aiohttp_session() + + with patch("solcast.aiohttp.ClientSession", side_effect=create_mock_session): + run_async(test_api.solar.fetch_pv_forecast()) + + label = "available" if with_ensemble else "missing" + expected = {"create_pv10": not with_ensemble, "calibrate_band": with_ensemble} + if captured != expected: + print("ERROR: ensemble {}: pv_calibration was called with {}, expected {}".format(label, captured, expected)) + failed = True + + # No generation history in this fixture, so P50 is uncalibrated and a created band is the fixed 0.7 / 1.3 + expected_ratios = (0.5, 1.5) if with_ensemble else (0.7, 1.3) + forecast = test_api.dashboard_items.get("sensor.predbat_pv_today", {}).get("attributes", {}).get("detailedForecast", []) + checked = 0 + for entry in forecast: + if entry["pv_estimate"] > 0.5: + checked += 1 + got = (entry["pv_estimate10"] / entry["pv_estimate"], entry["pv_estimate90"] / entry["pv_estimate"]) + if abs(got[0] - expected_ratios[0]) > 0.02 or abs(got[1] - expected_ratios[1]) > 0.02: + print("ERROR: ensemble {}: published band at {} is {:.2f}x / {:.2f}x of P50, expected {}x / {}x".format(label, entry["period_start"], got[0], got[1], expected_ratios[0], expected_ratios[1])) + failed = True + if not checked: + print("ERROR: ensemble {}: no published forecast slots to check".format(label)) + failed = True + finally: + test_api.cleanup() + + return failed + + def test_pv_calibration_no_history_not_zeroed(my_predbat): """ Regression test: when there is no valid historical data (e.g. all days excluded as @@ -6500,6 +6849,9 @@ def run_solcast_tests(my_predbat): failed |= test_pv_calibration_cap_applied_to_planner_data(my_predbat) failed |= test_pv_calibration_cap_pv10_never_exceeds_cap_or_p50(my_predbat) failed |= test_pv_calibration_cap_published_pv10_matches_planner(my_predbat) + failed |= test_pv_calibration_off_band_built_from_raw(my_predbat) + failed |= test_pv_calibration_keeps_provider_band(my_predbat) + failed |= test_fetch_pv_forecast_open_meteo_uses_ensemble_band(my_predbat) failed |= test_pv_calibration_no_history_not_zeroed(my_predbat) failed |= test_pv_calibration_no_history_ceiling_clips_raw(my_predbat) failed |= test_pv_calibration_raw_exceeds_ceiling_warns(my_predbat) diff --git a/docs/apps-yaml.md b/docs/apps-yaml.md index 8cc9b7ade..fb10b9e59 100644 --- a/docs/apps-yaml.md +++ b/docs/apps-yaml.md @@ -1957,7 +1957,11 @@ source changes. Do not judge the accuracy of the new source until the settling p [Open-Meteo](https://open-meteo.com/) is a free, open-source weather API that provides solar irradiance forecasts with no API key required. Predbat fetches the Global Tilted Irradiance (GTI) for each array and converts it to a power estimate using a PVWatts cell-temperature model. -Ensemble members are used to derive a PV10 pessimistic estimate alongside the central PV50. +Ensemble members are used to derive the PV10 pessimistic and PV90 optimistic estimates alongside the central PV50. +For each hour Predbat takes the ensemble's 10th and 90th percentiles as a ratio of the ensemble's own median, and applies those ratios to the PV50, +so the gap either side of PV50 is wider when the weather models disagree and narrower when they agree. +When PV calibration is on, PV10 and PV90 are scaled by the same calibration as the PV50. +If the ensemble data cannot be downloaded, PV10 and PV90 are created from the worst and best of your recent days instead. You can define one or more rooftop arrays by providing a list; they will be summed automatically. diff --git a/tools/debug-journal.md b/tools/debug-journal.md index 7df6801a2..fa3766ed7 100644 --- a/tools/debug-journal.md +++ b/tools/debug-journal.md @@ -109,7 +109,7 @@ Grep for the named symbol rather than trusting a line number. | Sigenergy (`sigenergy.py`) | Lifetime/history totals are cumulative server-side and reset around EU midnight, which showed up as an overnight dip; `fetch_history_totals` applies a monotonic clamp. "Energy totals went backwards overnight" starts here. Separately, a user `apps.yaml` override of `inverter.has_reserve_soc: true` (stock `SIG` default is `False`, deliberately — reverted in `5bfc5c80` after #2873/#3124) bypasses two guards that normally make reserve writes inert for Sigenergy (`execute.py:947-950`, `inverter.py:547-549`), so `adjust_reserve()` writes straight to `number.sigen_plant_ess_discharge_cut_off_state_of_charge`. Confirmed from a reporter's log: zero occurrences of the "Inverter does not support reserve" disable message the first guard would normally log, plus 56 reserve writes landing on that register (GH#4728). The stock template's `reserve:` line also points at that same entity rather than `ess_backup_state_of_charge`, which is what makes the override immediately destructive rather than merely inert. `Applying mode=eco ... duration=720min` is emitted only by the no-window fall-through branch (`sigenergy.py:2199`), so seeing it while a plan export window is live means the component's control snapshot never saw the window. Two code-verified routes to that, both still live: with `sigenergy_automatic: False`, `automatic_config()` - the only wiring of the component's control entities into Predbat - never runs, so Predbat's plan writes land on its own simulated dummy entities (`inverter.py:539-545`) and a `True == True` read-back is Predbat comparing against itself rather than reading a register; and `fetch_controls` runs on the first tick only (`if first:`, `sigenergy.py:2658`), so a single missed `call_service` event leaves the component's internal state permanently out of step while Predbat's write-skip comparison guarantees the healing write is never re-issued (GH#4894, probe-verified in `test_sigenergy.py`). `_publish_mqtt` logged the live `accessToken` on every command until PR #4926 added `SigenergyAPI.redact()`. The docs template still writes `number.sigen_plant_grid_import_limitation` to 0 in both freeze branches (`docs/inverter-setup.md:1936-1940` and `:1971-1975`), and Sigenergy's own EVAC/EVDC chargers are EMS-throttled to the plant import limit - so the block lands exactly when Predbat asks for Freeze Charging to hold for a car (GH#4911). | `sigenergy` | | AlphaESS (`alphaess.py`) | Once any discharge window is configured (`ctrDis=1`), AlphaESS firmware allows discharge *only* inside that window — outside it the battery is charge-from-PV-only, with no self-consumption fallback (AlphaESS's own documented behaviour, per GH#4701). Predbat's generic handling of what happens outside a configured discharge window (`execute.py`) has no AlphaESS-specific case and assumes normal demand mode resumes there, which holds for other brands but not this one — the next best export window Predbat schedules can be hours away, leaving the battery locked out of discharge until then (GH#4723). A separate write-ordering bug, **fixed in PR #4776 (merged 2026-08-27)**: the periodic API commits a schedule in stages — window, then enable switch, then target SoC — pressing the write button after each, so the *first* commit of a cycle always carried the *previous* cycle's target while `alphaess_min_write_interval` (default 300s) held the correcting write back for the rest of that pacing window; a manual charge on an already-live slot could run at the stale target for up to five minutes (GH#4769, confirmed from the reporter's log, not inferred). Also: the legacy `/updateDisChargeConfigInfo` endpoint's `batUseCap` field (the reserve/export-SoC value) has no floor on the Predbat side (`alphaess.py:1103`/`:1143`), while the periodic path clamps it to AlphaESS's documented `[10,100]` range — a reserve/export-SoC entity set below 10 gets every legacy write rejected with an undocumented `10001` (GH#4748); that errno is the signature to recognise this class by. **The single most important thing to know here, documented on `main` since PR #4744: the AlphaESS Open API cannot control export at all, so Predbat cannot make it export.** There is no forced-export, working-mode or dispatch endpoint - only a grid-charge window with a target SoC, a discharge window with an SoC floor, or both together on entitled systems. Force Export therefore exports nothing beyond genuine solar surplus, and Freeze Export comes out byte-identical to Demand mode because `gridCharge` gates only *timed grid charging* and nothing can stop the battery charging from solar. The documented answer is to set `select.predbat_mode` to `Control charge` on AlphaESS; left on `Control charge & discharge` Predbat plans exports that never happen *and* programs the discharge window up to `plan_interval_minutes` early, which on permission-window semantics bars the battery from covering house load until the window opens (that is GH#4723's mechanism). Forced export is reachable over local Modbus, never from the cloud API. Separately, PR #4867 changed how a hold is expressed: Predbat now writes a synthetic **enabled 10% target at 100 W** on a stable daily window (`ALPHAESS_HOLD_SOC`/`ALPHAESS_HOLD_POWER`, `alphaess_const.py:183-184`) rather than the old zero-rate hold, because zero power disables charging in Predbat and AlphaESS may discard such a profile. A genuine grid charge is not replaced by it (`alphaess.py:1106`). Note the consequence, raised in review and not tested on hardware: unlike a zero-rate hold this profile is not energy-neutral by construction, so an install sitting below 10% SoC has a standing 100 W charge request while the hold is in force. | `alphaess_control` | | Ohme (`ohme.py`) | A vendored, version-pinned copy of `dan-r/ohmepy` (`ohme.py:33`, `VERSION`). GH#4719 (2026-08-25) found it several minor versions behind upstream with the control routes (max-charge, session rule) pointing at a withdrawn `/v1/chargeSessions/{id}/rule` — a 404 with Spring's `NoResourceFoundException` — while GET/pause/resume/approve kept working since those stayed on v1 upstream. **Fixed on `main` since** (confirmed 2026-08-27): the vendored client now calls the v2 routes upstream moved those endpoints to. A reporter on an older Predbat version seeing control-only failures (GETs/pause/resume fine, target time/percent/preconditioning silently not applying) is hitting this, not a new bug. Separately, and **still live** (GH#4952): `slot_list()` derives slot energy as `watts * hours` (`ohme.py:124`), treating `watts` as a sustained rate when the observed 16A -> 10A -> 6A taper shows it is a per-slot cap - a reporter measured 48.49 kWh modelled against 19.02 kWh of real charge, 2.55x. Ohme publishes `estimatedSoc`, which would give the true delta, and `ohme.py` never reads it anywhere. The same slots are published as dispatches with `location: "AT_HOME"` and no `source` (`ohme.py:670`), so `rate_add_io_slots()` stamps the whole plug-in-to-target block at the low rate. Note the existing tests assert the current `watts * hours` semantics, and `estimatedSoc` is itself extrapolated when the car has no SoC telemetry - so neither side is a drop-in swap. See the car-charging row for why the `predict()` clamp has to be trimmed *after* this, not before. | `ohme` | -| Solcast (`solcast.py`) | `max_kwh` — the array-capacity ceiling used to sanity-check the forecast — is initialised to `9999` (`solcast.py:1339`) and is only ever reassigned on the Forecast.Solar and Open-Meteo branches. On the direct-API and HA-sensor Solcast paths it stays `9999`, so the "Raw forecast exceeds the array ceiling" warning (`solcast.py:1191`) can never fire for Solcast users — confirmed by reading the assignments (GH#4730). That warning is exactly the diagnostic an "impossible PV predicted" report needs, so its absence isn't evidence the ceiling wasn't exceeded. Branch order at `solcast.py:1368` also means a configured `solcast_api_key` always wins over `pv_forecast_*` HA sensors, so comparing Predbat's number against the HA entity compares against the wrong source when both are set. Two more. `solcast_poll_hours: 4.8` is silently truncated to 4: component args come from `COMPONENT_LIST`'s int defaults via `get_arg()`, which does `int(float(value))` (`userinterface.py:266`), and GH#4441's fix lives on the CONFIG_ITEMS route that component args bypass - the reporter's log showed real fetches at exactly 4h and 18x `code 429` against a 10/day hobbyist limit, while `docs/apps-yaml.md` recommends 4.8 for two-array accounts (GH#4925). And `publish_pv_stats()` reads the shared, briefly-rewound `midnight_utc` live (GH#4804) - see "The shared clock is rewound for ~0.8s every hour" above, which has since been confirmed as a real bug class by a separate merged fix. PV calibration self-reference (GH#5116, **fixed in PR #5121** — mechanism kept for older logs): `sensor._pv_forecast_h0` publishes the **calibrated** `power_nowCL` as its state whenever calibration is on, with the raw value only in the `now` attribute, so anything reading the state history gets calibrated output — and calibration itself used to read that history as its "past forecast" while applying its factors to the raw series, which settles the factor at **√(actual/raw)** and corrects only ~half the bias (logged 0.9073x against a measured 0.82). #5121 replaced the read with a dedicated `sensor._pv_forecast_h0_uncalibrated`, published unconditionally and never scaled by `pv_scaling`; `pv_forecast_history()` reads it (scaled by the *current* `pv_scaling`, so a change takes effect over the whole window at once), falls back per-point to the h0 `now` attribute and then the h0 state for points predating the sensor (`now` was added in the same commit that made the state calibrated, so a point without `now` recorded the raw state), and the web chart plots the uncalibrated series. The h0 state-vs-`now` distinction stays live for other consumers. Two general facts from the same PR's review: `history_attribute(attributes=True)` drops any point whose attributes lack the key unless `fallback_to_state=True` (the state path filters `unavailable`/`unknown`; the attribute path only skips them inside the fallback read — `utils.py`), and the DB mirror returns rows with `attributes = {}` on JSON decode failure (`db_engine.py` — the row is still appended), so an empty attribute read on a `db_primary` history does not prove the data predates the attribute. Anything comparing h0 attributes against Predbat's internal PV series must also account for `pv_scaling`, which sits on one side only (`now` is pre-scaling, the internal series post-scaling). Test-fidelity trap: a calibration test that patches `history_attribute_to_minute_data` cannot see which series calibration learned from — the real path needs a `get_history_wrapper` mock with HA-shaped entries wrapped in `patch_now_utc_exact()`, because `now_utc_exact` is real wall-clock while `minutes_now`/`midnight_utc` are frozen; `prune_today`'s default `group=15` also drops history points closer than 15 minutes apart. **Open-Meteo shading is client-side (GH#5224):** the Open-Meteo `v1/forecast` API has no `horizon` parameter (closest solar-geometry options are `tilt`/`azimuth` for GTI), so any horizon feature — Predbat's or a user asking for one — must be applied client-side to the downloaded irradiance, which is exactly what the HA Open-Meteo Solar Forecast integration (rany2) does; Predbat's existing knob is the static per-array monthly `shading_factors` applied in `gti_hourly_to_period_kwh()` (`solar_model.py`), which cannot express geometric blocking. *Suspected, not verified:* that integration's published attribute names could not be confirmed compatible with Predbat's HA-sensor PV path (`detailedForecast`/`forecast` with `pv_estimate` entries), so do not recommend pointing Predbat at its sensors as a workaround without checking the entity shape. | `solcast` | +| Solcast (`solcast.py`) | `max_kwh` — the array-capacity ceiling used to sanity-check the forecast — is initialised to `9999` (`solcast.py:1339`) and is only ever reassigned on the Forecast.Solar and Open-Meteo branches. On the direct-API and HA-sensor Solcast paths it stays `9999`, so the "Raw forecast exceeds the array ceiling" warning (`solcast.py:1191`) can never fire for Solcast users — confirmed by reading the assignments (GH#4730). That warning is exactly the diagnostic an "impossible PV predicted" report needs, so its absence isn't evidence the ceiling wasn't exceeded. Branch order at `solcast.py:1368` also means a configured `solcast_api_key` always wins over `pv_forecast_*` HA sensors, so comparing Predbat's number against the HA entity compares against the wrong source when both are set. Two more. `solcast_poll_hours: 4.8` is silently truncated to 4: component args come from `COMPONENT_LIST`'s int defaults via `get_arg()`, which does `int(float(value))` (`userinterface.py:266`), and GH#4441's fix lives on the CONFIG_ITEMS route that component args bypass - the reporter's log showed real fetches at exactly 4h and 18x `code 429` against a 10/day hobbyist limit, while `docs/apps-yaml.md` recommends 4.8 for two-array accounts (GH#4925). And `publish_pv_stats()` reads the shared, briefly-rewound `midnight_utc` live (GH#4804) - see "The shared clock is rewound for ~0.8s every hour" above, which has since been confirmed as a real bug class by a separate merged fix. PV calibration self-reference (GH#5116, **fixed in PR #5121** — mechanism kept for older logs): `sensor._pv_forecast_h0` publishes the **calibrated** `power_nowCL` as its state whenever calibration is on, with the raw value only in the `now` attribute, so anything reading the state history gets calibrated output — and calibration itself used to read that history as its "past forecast" while applying its factors to the raw series, which settles the factor at **√(actual/raw)** and corrects only ~half the bias (logged 0.9073x against a measured 0.82). #5121 replaced the read with a dedicated `sensor._pv_forecast_h0_uncalibrated`, published unconditionally and never scaled by `pv_scaling`; `pv_forecast_history()` reads it (scaled by the *current* `pv_scaling`, so a change takes effect over the whole window at once), falls back per-point to the h0 `now` attribute and then the h0 state for points predating the sensor (`now` was added in the same commit that made the state calibrated, so a point without `now` recorded the raw state), and the web chart plots the uncalibrated series. The h0 state-vs-`now` distinction stays live for other consumers. Two general facts from the same PR's review: `history_attribute(attributes=True)` drops any point whose attributes lack the key unless `fallback_to_state=True` (the state path filters `unavailable`/`unknown`; the attribute path only skips them inside the fallback read — `utils.py`), and the DB mirror returns rows with `attributes = {}` on JSON decode failure (`db_engine.py` — the row is still appended), so an empty attribute read on a `db_primary` history does not prove the data predates the attribute. Anything comparing h0 attributes against Predbat's internal PV series must also account for `pv_scaling`, which sits on one side only (`now` is pre-scaling, the internal series post-scaling). Test-fidelity trap: a calibration test that patches `history_attribute_to_minute_data` cannot see which series calibration learned from — the real path needs a `get_history_wrapper` mock with HA-shaped entries wrapped in `patch_now_utc_exact()`, because `now_utc_exact` is real wall-clock while `minutes_now`/`midnight_utc` are frozen; `prune_today`'s default `group=15` also drops history points closer than 15 minutes apart. **Open-Meteo shading is client-side (GH#5224):** the Open-Meteo `v1/forecast` API has no `horizon` parameter (closest solar-geometry options are `tilt`/`azimuth` for GTI), so any horizon feature — Predbat's or a user asking for one — must be applied client-side to the downloaded irradiance, which is exactly what the HA Open-Meteo Solar Forecast integration (rany2) does; Predbat's existing knob is the static per-array monthly `shading_factors` applied in `gti_hourly_to_period_kwh()` (`solar_model.py`), which cannot express geometric blocking. *Suspected, not verified:* that integration's published attribute names could not be confirmed compatible with Predbat's HA-sensor PV path (`detailedForecast`/`forecast` with `pv_estimate` entries), so do not recommend pointing Predbat at its sensors as a workaround without checking the entity shape. **Band built from the wrong series with calibration off (GH#5345):** on a `create_pv10` source (Open-Meteo, Forecast.Solar) with `metric_pv_calibration_enable` off, `pv_calibration()` built `pv_forecast_minute10`/`90` and the published `pv_estimate10`/`90` from the **calibrated** series and only swapped P50 back to raw at its final `return`, so P10 could exceed P50 and P90 fall below it, and `get_cloud_factor`'s (ΣP50 − ΣP10) gap was arbitrary - reported as "the saw tooth has gone". Signature in `sensor._pv_today` `detailedForecast`: `pv_estimate10 / pv_estimateCL` and `pv_estimate90 / pv_estimateCL` constant in every slot while `pv_estimateCL / pv_estimate` varies. Dead end: it is not Open-Meteo's ensemble P10 being "confident" - at the time that download was fetched and then overwritten by `pv_calibration()`, because every Open-Meteo branch set `create_pv10`; the same change makes `fetch_pv_forecast()` keep an ensemble band (`create_pv10` False, `calibrate_band` True) and fall back to the history-based band only when the ensemble download fails. The band is the ensemble's P10 and P90 **as ratios of the ensemble's own median**, applied to the deterministic P50, not the ensemble's absolute values: the P50 comes from the best-match deterministic call and the spread from `icon_seamless`, two different model runs that can disagree on a whole day's level - live on 2 Oct 2026 the absolute ensemble P10 sat above the deterministic P50 for a day three ahead (clamped, leaving a 2% downside) and at 0.28 of it elsewhere. Known weakness of the ratio: the deterministic forecast is re-issued between ensemble runs and moved one site's next-day total from 1635 to 3887 Wh/m2 in ten minutes, above the ensemble's own P90, and P50 x (P90/median) then overshoots anything the sky can deliver - only the `1.2 x kWp` power ceiling bounds it, there is no clear-sky cap. So an implausibly high `pv_estimate90` on an Open-Meteo site maps here. Test trap from that work: the solar test mock returns the first response whose key is a substring of the URL, and `api.open-meteo.com` is a substring of `ensemble-api.open-meteo.com`, so a test registering both in that order silently serves the forecast body to the ensemble call and exercises the no-ensemble path. The in-function `enabled_calibration` flag means "enough history", not the user switch, so the fixed 0.7/1.3 band does not apply with the switch off. Routine SolarAPI Info lines are not in shipped logs; read the `pv_today`/`pv_forecast_h0` attributes instead. | `solcast` | | HA write/verify (`ha.py`, `inverter.py`) | `get_state(refresh=True)` is a no-op in a normal HA add-on install: it only re-reads when `not self.ha_key`, and `ha_key` is set in that case, so verification reads the websocket cache, never a fresh value (`ha.py:798`, since PR #2342). Separately, `write_and_poll_value()` treats `domain == "sensor"` as Predbat-owned and POSTs `/api/states/` directly (`inverter.py:2155`, `ha.py:1071-1077`) instead of calling the integration. Probed empirically across all four write-path/domain combinations (GH#4738): a control entity that resolves to a `sensor.*` domain by mistake gets a direct state overwrite that reads back as success, with zero corresponding lines in the real integration's log. `call_service_wrapper()`'s return value used to be discarded at every `inverter.py` write site, so a rejected HA service call (e.g. a Modbus "Illegal Function" error) never reached the verify logic. That is no longer true everywhere: PR #4878 made `call_service_template()` check it (`inverter.py:2817`) and log "was not accepted, it will be retried next cycle". The other eleven `call_service_wrapper()` call sites in that file still discard it, so check the specific write path before repeating the general claim. Later triage extended this row rather than replacing it. `write_and_poll_switch()` builds its service name as `domain + "/turn_..."` straight from the entity's own domain with no validation (`inverter.py:2102`), and `adjust_inverter_mode()` routes every `has_ge_eco_toggle` inverter through it - so an `inverter_mode` pointed at a `select.*` entity produces `select.turn_on`, an HA-invalid service retried 10x at 10s per attempt, and the string coercion just below (`inverter.py:2110-2111`) maps any select state to False, so the force-export direction reports success having written nothing (GH#4909, still live — though on 3-phase GEC the real problem is one stage deeper: no eco register exists at all, and the write path is skipped with `No entity_id for ECO Toggle`; see the GE Cloud row). `adjust_force_export`/`adjust_charge_window` format their times at plan precision with no snapping (`TIME_FORMAT_HMS`, `inverter.py:2608`/`:2615`), so against an integration whose select only offers 15-minute options (Solar Assistant/Deye) Predbat writes e.g. `20:55` and `write_and_poll_option` retries the impossible value 10x every cycle; nothing anywhere reads a select's `options` attribute, and `charge_time_entity_is_option` only decides *how* an entity is written, never which values are legal. The fix precedent is already in tree - AlphaESS's `snap_time_grid()` (`alphaess_const.py:215`), snap start forward and end backward with a collapse check (GH#4947). A second, independent way plan-precision times fail on the same select path: `write_and_poll_option()` only strips the seconds from an 8-char `HH:MM:SS` write when the *previous* state is a healthy 5-char `HH:MM:SS` read — the read feeding it returns `None` for an entity missing from the state cache (`ha.py:817`) or the literal `"unknown"`/`"unavailable"` when degraded (`ha.py:815`), either skips the strip and the raw `HH:MM:SS` goes to `select_option`, which HA rejects; all 10 retries re-read and fail identically, and the log reads `didn't complete got unknown` (GH#5009, probe-verified on main). That degraded-read shape can also be **permanent**: an SA slot select can report `unknown` for days — every write read-back `unknown`, zero verified writes, across an SA rebuild and both native-API and MQTT transports — so both symptoms (read path `unable to read Export window - ... returned no data` every cycle via `time_string_to_stamp("unknown")` → `None`, and the write path above) fire continuously; state reporting varies by SA backend (GH#5214). **A user edit of `config.py`'s INVERTER_DEF `charge_time_format` to anything ≠ exactly `"HH:MM:SS"` is a masking workaround, not a fix (GH#5214):** the `!= "HH:MM:SS"` branch (`inverter.py`) swaps apps.yaml entities for Predbat's own `sensor.predbat_{type}_{id}_*` dummies — reads parse Predbat's own echo and writes become direct `set_state` POSTs that never reach the inverter — so the log goes quiet while SA window control is silently severed. Tells, all observed in #5214's attachments: a `Creating dummy entity sensor.predbat_{TYPE}_{id}_charge_start_time` line in the log (`create_entity()` only logs when the entity is missing from HA state, so its presence proves the ≠ branch ran); the debug yaml's live `args` naming dummy ids while `args_from_apps_yaml` still lists the real `select.*` entities; and warnings stopping after a user restart then resuming after an addon update (which overwrites the edited `config.py`, wiping it). Also: pre-#3533 SA shipped `charge_time_format: "S"` (dummy time entities, no SA select wiring), so "SA time warnings started after updating" can mean the user crossed the #3533 template wiring rather than a format regression. The fix that serves both is reading the select's `options` attribute — nothing in the codebase does today. Six of the eight `adjust_*` writers discard the write helper's boolean and fire `call_notify()`/`mqtt_message()` regardless, and `rest_setReserve()` has the same gap (GH#4845). `call_service_template()` used to record its dedup hash *before* the call, so a silently failed `select_option` was skipped as "previously called" on every later cycle; fixed (it now records only on success and drops the record on failure), though it still deliberately returns True so callers cannot downgrade a freeze into a stop (GH#4876). From that same report, a false-alarm class worth recognising: a `didn't complete got X` where X is exactly the *previous* cycle's target, on an integration that republishes its entities slowly, is the verify window reading HA's cache - the write had already landed by the next cycle. That window is arithmetic, not a constant: up to `INVERTER_MAX_RETRY` (10) writes, each polled through one `inv_write_and_poll_sleep` budget with a doubling read interval - **~40s on GS, ~100s on GE/GEC/GEE** (`write_and_poll_sleep` is 10 for those types, 4 for GS). **Per-type since PR #5297 (merged 2026-09-29):** `INVERTER_DEF[type]["write_max_retry"]`/`["write_backoff"]` (consumed by `_write_attempts()`/`_write_backoff_result()`), and currently only `GWMQTT` deviates - 3 attempts, 10s apart - plus a per-control degraded state for those types: a write of the *same* target failing `INVERTER_WRITE_BACKOFF_FAILURES` (2) times in a row drops to one attempt per `INVERTER_WRITE_DEGRADED_INTERVAL` (300s) until a write verifies, the read-back matches or a different target arrives (full ladder restored at once); a degraded control still reads back, logs and records the failure every call - nothing is skipped silently. Every other type keeps 10 attempts and its existing sleep. A settling bounce that outlasts it is a second false-alarm shape (GH#5130): a HA `number.*` entity can step through intermediates (0 → a float → the settled int) for tens of seconds after a timed-slot write (~44s observed on solax_modbus), so a `didn't complete got ` whose value sits *between* the old and target values is a settling read-back, not a failed write - it self-heals next cycle when the initial read matches. The bounce reads feed only `value_matched()` (keep polling vs rewrite) and only the final read reaches the control ledger (`record_write` on match, `clear` on failure), so the worst case is a redundant same-value rewrite plus the false warning; the in-repo precedent for re-reading after settling is Solis Cloud's `verify_settle_seconds` re-read. And the discharge-stop fallback (GH#5094, still live): `adjust_charge_immediate()` issues a "stop discharge" service step before the charge/freeze step, and when `discharge_stop_service` is not configured it falls back to `charge_stop_service` with `domain="discharge"` - a documented fallback dating to #1614. On Tesla service-hook setups `charge_stop_service` maps to `self_consumption` + backup reserve 0, which on a Powerwall is **active discharge to load** (the EV charger counts as load), not a stop - so every freeze/hold cycle Predbat sends "discharge normally" and then re-commands "hold" ~2s later, and Tesla occasionally applies the first and drops the second, so the battery discharges into the car until the next 5-minute cycle. Verified from the reporter's log: the exact pair fires from the carHolding branch ("Disabling battery discharge whilst car 0 is charging" → `adjust_charge_immediate(soc_percent, freeze=True)`); the reporter had commented out `discharge_stop_service` (and `discharge_start_service`) when disabling export control, which is what exposed the fallback. Workaround: define `discharge_stop_service` in apps.yaml with commands that actually stop discharge (for a no-export setup, the same commands as `charge_freeze_service`). Caveat on the reporter's own `repeat: False` idea: enabling the service dedup means accepted calls are not re-sent (see the GH#4876 entry above - HA accepts the call but Tesla firmware can still drop it), so dropped freeze commands would never be retried. Both this and the fixed export clobber below stem from the same property of the Tesla template set: `charge_stop_service` and the freeze/discharge templates all write the same `operation_mode` select, and the templates mean opposite things on this hardware. **A sibling of the same family, over the reserve register rather than the mode selects (GH#5300, triaged 2026-09-29 against d951201e, reporter on a hand-built apps.yaml):** inside one 5-minute pass the hook charge (`adjust_charge_immediate()` issues `charge_freeze_service`/`charge_start_service` via `call_service_template`, inverter.py) and `execute.py`'s reserve reset (`if self.set_reserve_enable and resetReserve: inverter.adjust_reserve(0)`) both touch the Powerwall's reserve and the reset has **no isCharging guard** - `resetReserve` initialises True whenever `set_charge_window or set_export_window` and is cleared only by the freeze / hold-charging / car / iBoost hold branches - while `adjust_reserve()` floors at `reserve_percent` and writes through `write_and_poll_value()`, so the reset always ends up holding the register, against hook writes that are fire-and-forget. A service hook that uses the reserve register as its charge signal loses that race on every charge-window cycle. The shipped template is immune by two mechanisms, both worth checking before triaging any "Tesla reserve overwritten" report: `templates/tesla_powerwall.yaml` sets `has_reserve_soc: False` in the `inverter:` block (`fetch_inverter_data()` then force-disables `set_reserve_enable`/`set_reserve_hold`, logging `Note: Inverter does not support reserve - disabling reserve functions`), and with no `reserve:` arg wired Predbat dummies the arg (`create_missing_arg`/`create_entity`, inverter.py) so `adjust_reserve` writes land on a Predbat placeholder - the Sigenergy dummy-arg shape again. Discriminator for real-vs-dummy reserve wiring: the `Inverter N Current Reserve is X% ... and new target is Y%` and `Wrote X to reserve, successfully now Y` lines name the register, not the entity - a non-zero current read means the `reserve:` arg resolves to a real wired entity; a dummy reads 0 and at an equal target prints `already at target` instead. Dump-config tells from the same report: a `set_reserve_enable` switch showing `value: null` in a debug dump can still resolve True (expert-mode-switched item, hidden and entity not created while expert mode is off), and unknown keys in the dump's `args:` mean a hand-built config - do not assume the shipped template. | `inverter` | | GE Cloud (`gecloud.py`) | Gateway fields can come back null. `merge_non_null` stops nulls overwriting good values, and the publish path guards null containers — GH#4656 was a null crash in that area. `number_event` clamps an out-of-range write silently and does not report the clamped value back, while `write_and_poll_value()` keeps comparing against the original pre-clamp target - a permanent false `had_errors` and an unbounded retry (GH#4826). PR #4828 clamped inside `adjust_reserve()` only; the shared write helper still has no min/max awareness and the other `inverter.py` write sites still pass unclamped targets (GH#4831). The error-code path has since been reworked (`9560c61a`, `d6e5998c`) after GH#4896 showed `success: false` nulled `data` before the code branch could read it, so codes GivEnergy documents as never succeeding on a repeat (-3/-4/-7) were retried the full budget and `message` was never surfaced. Separately, **fixed in PR #4954 (merged 2026-09-06)** - keep the mechanism for anyone holding an older log: for `GE`/`GEC`/`GEE` the max battery rate was taken solely from the `charge_rate` entity's `max` attribute, and the `elif "battery_rate_max"` branch below it was unreachable for those types. Percentage-rated models (the 3-phase units) have no absolute charge power register, so `ge_cloud_automatic` left `charge_rate` unset and the attribute read returned the 2600 W default; both the correct rate GECloud writes into `battery_rate_max` and the user's own apps.yaml value went unread, and every rate was clamped to 2600 W. On those models it also corrupted the writes, since the percentage registers are set as `new_rate / battery_rate_max_raw * 100` against a denominator 3.8x too small. The fix falls back to `battery_rate_max` only when `charge_rate` resolves to nothing (GH#4908). GH#4198 is the same code shape on SolaX and is not covered by that fix. A model-string parser gap sits above that path (GH#5136, still live): `get_max_inverter_rate_from_model()` (gecloud.py) matches only decimals (`\d+\.\d+`), so an integer-rated model such as `GIV-3HY-11` falls back to `max_charge_rate` - which the GE Cloud API can change independently (11000 → 9984 W observed) - and `publish_info()` reads only `max_charge_rate`, never the `max_discharge_rate` in the same payload. Distinct from GH#4908 (the absent-`charge_rate` 2600 W default). Start on any "GEC inverter rate is wrong" report by checking the model string for a decimal point; the test covers only decimal cases. `automatic_config()` binds `inverter_limit` to the resulting `_max_inverter_rate` sensors and `prediction.py` clips PV against `inverter_limit`, so the error reaches the plan. One API-shape trap on the site endpoint (`GE_API_SITE`, `site/{uuid}`, added for PR #4978): the published OpenAPI spec types `limits.import`/`limits.export` as a string with a null example and never shows a populated one, so the shape had to be learned from live accounts. A populated limit is `{'enabled': bool, 'power': {'watts': int, 'amps': float}}`, and **`enabled` is the thing that matters** - a site describes its connection's declared capacity in exactly the same structure as a curtailment, marked disabled, so `power.watts` alone tells you nothing about whether anything is being restricted. Site 64744 gave `{'import': {'enabled': False, 'power': {'watts': 46000, 'amps': 200}}, 'export': {'enabled': True, 'power': {'watts': 4500, 'amps': 19.6}}}` - a 200A supply whose export really is curtailed to 4.5kW - while site 46714's disabled 6kW export is its declared capacity and no restriction at all. A fleet survey confirmed the flag across accounts, so only an enabled limit is applied; an enabled zero is a real zero-export connection. A site with no limit reports a null import/export. For a fleet survey use `GET /v1/site`, the list endpoint, which returns every site's `limits` and the inverters under it (with `info.max_discharge_rate`) in one response - no second call per site. Two GEC triage tools and one clobber trap (GH#5017). The component publishes an entity for every `/settings` entry with no name filter (`async_get_inverter_settings()`/`publish_registers()`, `regname_to_ha()` is plain lowercasing), so **absence of an entity in the reporter's debug yaml is proof the API never listed the register** — "the app can control it" does not imply the settings API exposes it, because the app/GivTCP write via local control paths rather than the `/settings` list (a GIV-3HY-20 had slot 1/2 lower-SoC registers absent while 3..10 existed). Caveat: the settings list is fetched once per process start (`register_list` cache) and only the parsed flags are logged, so the debug yaml is the only observable. To capture the raw register list, `python3 apps/predbat/gecloud.py --api-key --write-entity number.probe_nothing 0` makes `test_gecloud_direct()` print the full "Available entities/registers" list with raw cloud names and setting IDs and performs no write — but the harness's startup pass also runs `enable_default_options()` against the real inverter (floors reset, unused windows zeroed), so say so when asking a user to run it. Clobber trap: the missing-feature branch `set_arg("discharge_target_soc", None)` *deletes* the key, including an explicit apps.yaml entry the user wrote; `set_arg_auto(..., overwrite=False)` (component_base.py) is the in-repo pattern for letting an explicit value win — check the `has_*` detection branch on any "automatic config deleted my setting" report. The `inverter_mode` half of this class was fixed in PR #5059 (`885c637d`) — it now binds via `set_arg_auto(..., overwrite=False)` like `export_limit` — but the other missing-feature deletes are still plain `set_arg(..., None)` on main: `pause_mode`, `pause_start_time`, `pause_end_time`, `discharge_target_soc`, `charge_rate_percent`, `discharge_rate_percent`, `givtcp_rest`. A companion test-fidelity trap (GH#5056, verified on main): the component test mocks (`MockBase.set_arg` in `test_ge_cloud.py`; same shape in `test_ohme.py` and `test_alphaess_api.py`, whose mock `set_arg_auto` also writes unconditionally) store the value instead of mirroring the real `set_arg()`'s None-deletes, and their `args_from_apps_yaml` is empty so `set_arg_auto()` falls through to the plain path — so the "feature not detected" assertions (`config_args.get(...) is None`) pass identically whether the code stored None or deleted the key, and a green suite cannot distinguish fixed from unfixed. Any test for this class must populate `args_from_apps_yaml` and assert the manual value survives — #5059's Test 10 in `test_ge_cloud.py` is now the in-repo precedent. And the deeper half of GH#4909 (still live): the 3-phase GEC class exposes **no** eco/demand/mode register in `inverter/{sn}/settings` — verified by enumerating the `predbat_gecloud_*` entities in the reporter's debug dump — yet the `GEC` def sets `has_ge_eco_toggle: True` unconditionally (`config.py`) and auto-config maps `inverter_mode` via `build_entities("switch", ["enable_eco_mode"])` (gecloud.py), which emits nothing when the register is missing, so `adjust_inverter_mode()` logs `No entity_id for ECO Toggle` and returns without writing every plan cycle and freeze export silently no-ops (force-discharge paths work fine). Fix wrinkle for the maintainer: the flag is one per-inverter-type bool while `build_entities` works per-device. The freeze-export half of the same class (GH#5082, follow-up to GH#4909, code-verified on main): `inv_has_timed_pause` is probed at runtime (inverter.py flips it False when the `pause_mode` entity is absent/unavailable) but `inv_support_discharge_freeze` is read only from the static `INVERTER_DEF` entry (`config.py` sets it True for GEC), so `execute.py` never force-disables `set_export_freeze` for the eco-less/pause-less 3-phase class and a freeze-export plan can never be executed. Check both flags on any "freeze silently didn't work on GEC" report - and do not grep only for `No entity_id for ECO Toggle`: that warning comes from the eco branch, while freeze export on this class fails primarily through the pause/rate branch, so the eco warning being absent says nothing about the freeze path. Fix direction: mirror the `inv_has_timed_pause` presence-probe; the manual-freeze-export override path already degrades correctly when `set_export_freeze` is False (plan.py drops to demand). Hardware-reported on the GIV-3HY class: `charge_power_rate` limits AC (grid) charging only - `battery_power` stays at the full discharge figure while the charge register reads 1% - so the `adjust_charge_rate(0)` fallback in the freeze branch is a no-op for PV charging there; note `charge_discharge_with_rate` is False for GEC, so the first `adjust_charge_rate(0)` in each branch is skipped and the write only happens via the `has_timed_pause`-else fallback. Fix precedent for building a freeze from registers the device actually has: `inv_support_feedin_first` + Fox Cloud freeze export (PR #5038); the GIV-3HY exposes `enable_force_discharge` + `discharge_down_to_percent` + `battery_reserve_percent`, the raw material for the #4909/#5082 option-2 freeze. And since #5056 a manual `inverter_mode` apps.yaml override is silently deleted by automatic config, so "just map the eco toggle manually" is not currently viable either. The neighbouring **charge-side** gap (GH#5040: `scheduled_charge_enable` had no `enable_force_charge` candidate, so a 3-phase GEC that gates timed grid charge behind both charge switches logged real charge states while importing nothing) is **fixed in PR #5042** (`a7121135`, v9.0.2): auto-config now binds `scheduled_charge_enable` to `enable_force_charge` with `ac_charge_enable`/`enable_ac_charge` as fallbacks, and `enable_default_options()` holds `enable_ac_charge` on as a static enable when the force register exists. Keep the mechanism for pre-v9.0.2 logs, where the signature is a 3-phase GEC showing plan charge states against flat grid power with no warning anywhere. The #5042 gate itself had a second gap (**GH#5269, fixed in PR #5270, merged 2026-09-27** — keep the pre-#5270 signature): `enable_ac_charge` was written in exactly one place, `enable_default_options()` — first cycle after start-up, then once per 24h (`default_options_stamp`) — while Predbat's own `scheduled_charge_enable` write (`switch_event()`) checked nothing about the gate, so any external writer (an Axle VPP clean-up, the GE app, another integration) could clear `enable_ac_charge` and Predbat re-asserted it only on that 24h pass or a restart; "restarted and it worked" is part of the pre-fix signature, as is `enable_force_charge` on with `Charging target …%` for hours, SoC flat and no error. #5270 re-asserts the gate from `switch_event()` when a force-charge write succeeds (`ensure_ac_charge_gate`, with Axle's clean-up read as read-only for the 24h pass and a binding guard), so the "gate cleared between cache refreshes" residual window is the remaining maintainer call on that fix. So "3-phase GEC plans a charge, status says Charging, battery imports nothing" now has **two** signatures: pre-v9.0.2 = never bound (#5040), v9.0.2+ pre-#5270 = externally cleared and not re-asserted (#5269); on a post-#5270 system check whether the force switch Predbat drove is actually the `enable_force_charge` register. GE Cloud EMS plants (GH#5103, **fixed in PR #5109, merged 2026-09-24** — keep the mechanism for pre-#5109 logs): with an EMS discovered `polling_mode` is set False (`gecloud.py`), and the settings refresh used to re-read each battery inverter's `/settings` only on the first tick of the process — so the AC3 registers, including the DC-discharge slot windows, were read once and published stale for the rest of the run, while live status/meter polling kept looking normal. #5109 re-reads settings hourly (`SETTINGS_SLOW_REFRESH_SECONDS`, `gecloud.py`), refreshes the EMS and gateway devices every cycle, and runs `check_ems_inverter_slots()` on every snapshot — which now *warns* when a battery inverter's slot 1 is not the full-day window (the #3781 rule: `Inverter X slot 1 is not 00:00-23:59 ... it will override the EMS and can stop charging or discharging early`), once per episode — recorded in `ems_slot_warned` so a standing misconfiguration does not inflate the error counter on every refresh, so the non-00:00–23:59 slot 1 fault is a visible warning on current main and silent only pre-#5109 (pre-fix signature: plan/status look right but the batteries hold 0 W past a fixed wall-clock time every night). Under EMS auto-config Predbat still writes mode/schedule-enable/rates to the AC3s. Nuance: the startup read itself can still be skipped when the storage settings cache is under 10 minutes old (`settings_from_cache`), so even a restart is not guaranteed to re-snapshot the registers. **The option-validation parser used to crash-loop the whole component (GH#5217, fixed in PR #5218, merged 2026-09-26 — keep the signature for pre-#5218 logs):** `select_event()` and `publish_registers()` both `split("(")` a cloud `Value must be one of: (...)` string, so an option label containing `(` raised `ValueError: too many values to unpack (expected 2)` — the component does not die (`ComponentBase.start()` catches it), it *crash-loops*: `api_started` never set, automatic config never runs, no plan, `Error: GECloudDirect: too many values to unpack (expected 2)` every cycle. #5218 added the module-level `parse_validation_options()` (`gecloud.py`): split on the first `(` and strip only the final `)` (so inner brackets in labels survive), and `None` for a string with no `(` triggers a warning and falls back to the setting's `in:` validation-rule options instead of raising — the paren-less guard is required, not optional, because `split("(", 1)` alone fails the other direction. Which GE Cloud setting actually returns a multi-`(` validation string was never captured (the storage cache saved at the settings write would be the fixture to ask for). **`refresh_discovery()` is not device re-discovery, and GE Cloud re-discovers hourly since PR #5295 (GH#5294, merged 2026-09-29 - the symbols below were read against the pre-fix tree):** `ComponentBase.refresh_discovery()` only rebuilds the capability-catalogue *report* from data already in hand (a component opts in via `build_discovery()`) and never calls the component's API - the *similarly named* `refresh_devices()` is the thing that calls it. Pre-#5295 that device discovery (`async_get_devices()`, EV chargers, EMS/gateway selection, one-shot automatic config) ran exactly once inside `if first:` and `first` flips off permanently after the first successful run, so on any "GE Cloud didn't pick up my new inverter/charger" or "removed inverter still shows data" report a restart was the only mechanism - and a removed device kept its last good SoC/power feeding the plan (`merge_non_null()` hands back the previous reading on nulls and failed polls) and kept having registers written to. **Post-#5295:** `refresh_devices()` re-reads the device list hourly (`DEVICE_REFRESH_SECONDS`, 60*60, `gecloud.py`; a multiple of 60 because `run()` is called once a minute, and a cycle that finds a change polls and reads settings whatever else is due), `select_poll_devices()`/`apply_devices()` decide what is polled from the fresh list, and adopting a changed set drops every per-device store - `status`, `meter`, `info`, `settings`, `pending_writes`, `register_list`, the EV-charger stores, `ems_slot_warned` and `register_entity_map` - so a removed device stops publishing and being written to. The review round also: the change signature includes `battery_meters`, so a CT rewiring re-runs automatic config; a failed register-list fetch is retried on the next settings read (previously stored as `None` and never fetched again - every later read raised `TypeError`, newly reachable from rediscovery); a device read that would leave zero battery inverters is not adopted (the warning says restart if the hardware really was removed; charger-only sites stay quiet); `ge_cloud_data` returns to its pre-EMS value when the EMS leaves; the EMS slot-override warning re-arms when the EMS changes; and default options for a device discovered while read-only is set stay pending across cycles, applied once read-only ends. Caveats that survive the fix: the 5-day `last_updated` skip inside `async_get_devices()` treats a device silent over 5 days as non-functional wherever that runs - the hourly re-read included - and `merge_non_null()` still hands a *polled* device its last good SoC/power on nulls and failed polls. In-tree fix precedents (GE Cloud itself now among them): Fox re-fetches its device list every 24h (`FOX_REFRESH_STATIC`), SolaX gates plant/device-info re-reads with `data_is_due`, and `num_inverters` is re-read by the core every cycle so a re-run automatic config takes effect next plan. | `ge_cloud` | | Teslemetry / Powerwall (`teslemetry.py`) | Battery model is inferred from site `nameplate_power / battery_count`. A nameplate fallback combined with `inverter_hybrid: True` once produced a false 5 kW inverter limit and spurious morning export. Tariff writes push a whole TOU schedule every cycle, so a setting changed by hand in the Tesla app is reverted on the next cycle (GH#4600, GH#4610). Three later findings. `build_tariff()`/`_render_side` carve a *single* ON_PEAK interval priced with one scalar (`teslemetry.py:1117`/`:1036`/`:1074`), so no intra-window price gradient can exist and Tesla's own TBC is free to defer the whole export to the end of the window; the no-rate-data fallback branch also skips the boost entirely (GH#4887). The charge path sends no rate at all - `evaluate_schedule` (`teslemetry.py:539`) asserts mode `backup` plus reserve = charge target plus grid charging - so a slow charge ramp is Tesla firmware, not a Predbat write bug (GH#4892); Tesla now snaps a backup reserve of 81-99% down to 80% while Predbat still forwards any 0-100 target. The site_info limit mapping was reworked in PR #5276 (GH#5275, merged 2026-09-28): `inverter_limit` comes from `nameplate_power` (the Powerwall's own AC rating; `max_site_meter_power_ac` is the site's supply limit at the meter, not the inverter's), `export_limit` from `min_site_meter_power_ac` (kW, possibly fractional, with the ±1e9 "no limit" sentinel skipped and 0 published as a real 0 W limit), `inverter_limit_charge` at **5 kW per battery unit** capped at nameplate (Tesla reports no usable charge rating — a Powerwall 3 lists only expansion packs with every rating zero, and fleet data fits the rule), `soc_max` from the gateway's `nameplate_energy_watts` before the `battery_count` estimate, and `inverter_hybrid` now defaults from the Powerwall model (Powerwall 3 on, every other model off) with a `teslemetry_hybrid` override for a PW3 beside an existing string inverter. All of those are wired with `set_arg_auto(overwrite=False)`, so **an apps.yaml value wins** — on a pre-#5276 version `automatic_config()` overwrote a manually set `battery_rate_max` whenever `teslemetry_automatic` was on, which was the standing "my override does nothing" answer on this component. Since PR #4976 there is a Predbat-side lever for exactly that slow ramp: `teslemetry_tbc_control` — **on by default since PR #5188** (GH#5186; `DEFAULT_TBC_CONTROL = True` in `teslemetry.py`, mirrored in `COMPONENT_LIST`, with `teslemetry_tbc_control: False` as the opt-out) — switches `evaluate_schedule()` to `evaluate_schedule_tbc()`, which pushes a **signal tariff** (`build_signal_tariff()` - fixed 0/50/100p bands over the committed charge and export windows) so Tesla's own optimiser runs the charge and reaches the full rate reserve-driven charging cannot; mode goes to `autonomous` in every state, reserve becomes an actual reserve rather than a charge signal, and `_settable_reserve()` maps requests onto reserves the Powerwall honours - an 81-99% request now rounds **up** to 100 rather than being forwarded raw into the snap-to-80 band. The same PR stopped `plan.py` planning a manual freeze export on inverters that cannot do it. **Consequences of the default flip:** a Teslemetry user who never set the key is on the signal tariff with autonomous mode, so "the Tesla app shows 0p/50p/100p instead of my rates" or "my charge target isn't honoured" is the default path working, not a bug; and the 81-99% `set_reserve_min` limitation becomes default-on behaviour with it. (`MockTeslemetryAPI` still defaults `tbc_control = False` on purpose, so a green teslemetry test run exercises the real-rate path unless a test opts in.) And a separate scoping fact from the GH#5186 triage: `teslemetry_automatic` gates only `automatic_config()` — the scheduler emulator (`sync_tariff()` + `assert_device_state(evaluate_schedule(...))`) runs for every non-read-only user with no `automatic` check, so "Predbat changed my Tesla tariff/mode but I never turned on automatic" is that scoping, not a bug; `set_read_only` is the only switch that stops the writes. A related pricing bug in the same `build_tariff()` family was fixed in PR #4977: an export window ending at midnight priced the whole of tomorrow at peak. And `optimization_strategy: "economics"` *is* pushed with `tariff_content_v2` on every `set_tariff()`, unconditionally, since v8.49.0 / PR #4603 - a request to "add" it is asking for something already shipped (GH#4918). **GH#5157** (reporter's evidence, not independently probed — Powerwall 2, fw 26.26.4): the unit sat in the idle branch of `evaluate_schedule` and simply stopped acting on a standing device tuple for ~4h while `site_info` read back the *correct* configuration — drift invisible to every field the API exposes, with only a Predbat restart recovering it; the stall state is the one branch Predbat legitimately holds for hours, which is why transition-based self-heal never fires. A "Powerwall plateaued / stopped discharging until I restarted Predbat" report maps here first. **Implemented on main** (2026-09-19): `run()` re-asserts the whole device tuple — export rule, grid charging, reserve, mode, not the tariff — every `FORCED_ASSERT_SECONDS` (2h) with the write-on-change dedupe bypassed (`force=True`), the timer advancing only on a fully successful forced assert; the Info line `Teslemetry control drift-correction is transition-based ..., backed by a forced re-assert of the full device tuple every 120 minutes` names it, so on current main the stall is bounded at ~2h. **Freeze export is fully implemented in the planner and disabled for TESLA at exactly one point (GH#5225, enhancement):** `INVERTER_DEF["TESLA"]` declares `support_discharge_freeze: False` (`config.py`), read into `inv_support_discharge_freeze` (`inverter.py`), whose only consumer is the first-inverter lockstep block in `execute.py` that force-disables `set_export_freeze`/`set_export_freeze_only` (`Note: Inverter does not support discharge freeze - disabled`) — `EXPORT_MODE_FREEZE` never enters the export ladder, and PR #4976 additionally drops manual freeze-export windows to demand with a Warn. **Flipping the flag alone is not enough:** the component must also map the freeze-target discharge window to a device-side hold, or the plan's freeze assumption fails on the device — the non-TBC `evaluate_schedule()` writes the *raw* discharge target as the reserve, so a 99 target lands in the 81-99 band Tesla silently snaps to 80 (`_settable_reserve()` exists for exactly that and the non-TBC path doesn't route through it), while the default-TBC `evaluate_schedule_tbc()` discharge branch yields `pv_only` with the standing reserve but no hold — the battery still serves house load and drifts down. The device primitive exists in-tree: the TBC charge-hold tuple (`pv_only` + reserve 100 + grid charging off + autonomous) holds the battery while PV surplus exports, but a freeze *export* hold must sit at the current SoC (reserve at/above SoC through `_settable_reserve()`), not 100 — reserve 100 lets solar recharge the battery instead of exporting the surplus. Every `INVERTER_DEF` entry with `support_discharge_freeze: False` (the cloud-emulator types: SunsynkCloud, AlphaESSCloud, DeyeCloud, SolaxCloud, SolisCloud…) shares this two-part requirement — capability flag **plus** device-side freeze mapping — so enabling freeze export on any of them is a two-file change, not a one-line flag flip; each component's schedule emulator needs its own hold tuple designed against what its hardware honours (*suspected*, not verified per component). | `teslemetry` | From a2960a00e37d0208dd4765925afbec77d4e4aac8 Mon Sep 17 00:00:00 2001 From: CI Date: Fri, 2 Oct 2026 18:14:53 +0100 Subject: [PATCH 2/2] fix(solar): use a true median for the Open-Meteo ensemble ratios With an even number of usable members the nearest-rank pick took the lower of the two middle values, skewing both the P10 and P90 ratios. Take the mean of the two instead, and cover the even-member case. Co-Authored-By: Claude Opus 5.5 (1M context) --- apps/predbat/solcast.py | 5 ++++- apps/predbat/tests/test_open_meteo.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/apps/predbat/solcast.py b/apps/predbat/solcast.py index 19f583c50..7d6c09b94 100644 --- a/apps/predbat/solcast.py +++ b/apps/predbat/solcast.py @@ -306,7 +306,10 @@ async def download_open_meteo_ensemble_data(self, lat, lon, tilt, az): continue values.sort() gti_p10 = values[max(0, math.ceil(len(values) * 0.10) - 1)] - gti_p50 = values[max(0, math.ceil(len(values) * 0.50) - 1)] + # A true median: with an even number of usable members there is no middle one, so + # take the mean of the two either side rather than the lower of them + middle = len(values) // 2 + gti_p50 = values[middle] if len(values) % 2 else 0.5 * (values[middle - 1] + values[middle]) gti_p90 = values[max(0, math.ceil(len(values) * 0.90) - 1)] if gti_p50 <= 0: continue diff --git a/apps/predbat/tests/test_open_meteo.py b/apps/predbat/tests/test_open_meteo.py index f65d8b24f..671fe6300 100644 --- a/apps/predbat/tests/test_open_meteo.py +++ b/apps/predbat/tests/test_open_meteo.py @@ -69,11 +69,11 @@ def test_ensemble_returns_band_ratios(my_predbat): try: test_api.solar.open_meteo_forecast_max_age = 1.0 ensemble_data = _make_ensemble_response( - times=["2025-06-15T04:00", "2025-06-15T05:00", "2025-06-15T12:00"], + times=["2025-06-15T04:00", "2025-06-15T05:00", "2025-06-15T12:00", "2025-06-15T13:00"], members={ - "global_tilted_irradiance_member01": [0.0, 1.0, 400.0], - "global_tilted_irradiance_member02": [0.0, 2.0, 450.0], - "global_tilted_irradiance_member03": [5.0, 40.0, 480.0], + "global_tilted_irradiance_member01": [0.0, 1.0, 400.0, 300.0], + "global_tilted_irradiance_member02": [0.0, 2.0, 450.0, None], + "global_tilted_irradiance_member03": [5.0, 40.0, 480.0, 500.0], }, ) test_api.set_mock_response("ensemble-api.open-meteo.com", ensemble_data) @@ -101,6 +101,11 @@ def create_mock_session(*args, **kwargs): if result.get("2025-06-15T05:00") != (0.5, 2.0): print(f"ERROR: ensemble ratios at 05:00: expected (0.5, 2.0) with the P90 ratio capped, got {result.get('2025-06-15T05:00')}") failed = True + # 13:00 -> [300, 500] with one member missing: an even count has no middle member, so the + # median is the mean of the two middle values (400), not the lower of them + if result.get("2025-06-15T13:00") != (0.75, 1.25): + print(f"ERROR: ensemble ratios at 13:00: expected (0.75, 1.25) against a true median of 400, got {result.get('2025-06-15T13:00')}") + failed = True finally: test_api.cleanup()