Merge pull request #4726 from springfall2008/fix/dawn-detection-fix

fix(fetch): split charge windows at dawn again, unify low-power PV thresholds

Author: springfall2008

apps/predbat/config.py

@@ -1075,6 +1075,25 @@
         "enable": "set_charge_low_power",
         "default": 10,
     },
+    {
+        "name": "low_power_pv_threshold_w",
+        "friendly_name": "Low power mode PV threshold",
+        "type": "input_number",
+        "min": 0,
+        "max": 2000,
+        "step": 10,
+        "unit": "W",
+        "icon": "mdi:weather-sunny",
+        "default": 150,
+    },
+    {
+        "name": "set_charge_low_power_solar_full_rate",
+        "friendly_name": "Low power mode full rate in solar",
+        "type": "switch",
+        "icon": "mdi:solar-power",
+        "enable": "set_charge_low_power",
+        "default": True,
+    },
     {
         "name": "set_reserve_enable",
         "friendly_name": "Set Reserve Enable",

apps/predbat/const.py

@@ -85,22 +85,6 @@
 MAX_INCREMENT = 240 * 100 * 3 / 1000 / 60
 MINUTE_WATT = 60 * 1000
 
-# PV production (kWh) forecast across the remainder of a charge window above which low power charging is
-# abandoned in favour of the max charge rate. Throttling the charge rate while the sun is shining stops the
-# PV reaching the battery, the surplus is exported cheaply and the target is then made up with grid import,
-# which increases the cost of the plan over the full rate charge the planner costed the window at.
-LOW_POWER_PV_THRESHOLD = 0.1
-
-# Fraction of the peak forecast PV power above which a plan_interval_minutes bucket is classed as
-# "light" rather than "dark" when deciding where to split a charge window (calc_dawn). A charge window
-# otherwise built from a single long cheap-rate period spanning sunrise would apply LOW_POWER_PV_THRESHOLD
-# across the whole thing and abandon low power charging even for the still-dark hours before the sun is
-# up (#4557) - splitting at dawn keeps the dark portion as its own window, genuinely PV-free, so it stays
-# throttled. A fraction of that forecast's own peak, rather than a fixed Watts figure, scales with the
-# site - a fixed threshold picked for a typical system would be noise-level for a large array and
-# unreachable for a small one.
-LOW_POWER_PV_LIGHT_FRACTION = 0.1
-
 INVERTER_TEST = False  # Run inverter control self test
 
 # Sentinel values for an export window's target SoC/limit (export_limits_best and friends).

apps/predbat/execute.py

@@ -274,6 +274,8 @@ def execute_plan(self):
                             self.battery_temperature_charge_curve,
                             current_charge_rate=current_charge_rate / MINUTE_WATT,
                             pv_window_kwh=pv_window_kwh,
+                            low_power_pv_threshold_w=self.low_power_pv_threshold_w,
+                            solar_full_rate=self.set_charge_low_power_solar_full_rate,
                         )
                         new_charge_rate = int(new_charge_rate * MINUTE_WATT)
 

apps/predbat/fetch.py

@@ -32,7 +32,6 @@
     LOAD_FORECAST_HISTORY_MAX_DAYS,
     PREDBAT_MAX_CARS,
     CAR_CHARGING_LIMIT_UNCAPPED,
-    LOW_POWER_PV_LIGHT_FRACTION,
     CLOUD_WINDOW_MINUTES,
     CLOUD_ARRAY_MARGIN,
     PV_ARRAY_KWP_UNKNOWN,
@@ -905,6 +904,31 @@ def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling
             load_minutes = MinuteArray(load_minutes, size)
         return load_minutes, age_days
 
+    def fetch_pv_forecast_and_dawn(self):
+        """
+        Fetch the PV forecast, compute the dawn light/dark split from it, and publish the dawn
+        binary_sensor - as one call so the ordering that #4699 regressed on (the split must be
+        computed from a freshly-fetched forecast, not the empty dict fetch_sensor_data() resets it
+        to at the top of each cycle) can't drift apart again. Published unconditionally regardless
+        of whether import rates currently exist, so the sensor never goes stale.
+
+        Returns:
+            pv_light_dark: dict as returned by calc_pv_light_dark(), also stored on self.pv_light_dark.
+        """
+        self.pv_forecast_minute, self.pv_forecast_minute10, self.pv_forecast_minute90 = self.fetch_pv_forecast()
+
+        # Stored on self, not just local, so it can be used elsewhere rather than only by the
+        # window split below.
+        pv_light_dark = self.pv_light_dark = self.calc_pv_light_dark()
+        # "on" past dawn, "off" before it or when unclassified (combine_charge_slots off, or no
+        # PV forecast) - not the same as PV actually producing right now, see calc_dawn's docstring
+        self.dashboard_item(
+            "binary_sensor." + self.prefix + "_dawn",
+            state="on" if pv_light_dark.get(self.minutes_now) == 1 else "off",
+            attributes={"friendly_name": "Predbat is past dawn (light, not dark, in the low-power charge window split)", "icon": "mdi:weather-sunset-up"},
+        )
+        return pv_light_dark
+
     def combine_active_keep(self):
         """
         Combine the SOC keep floors (alerts, manual_soc) and ceilings (manual_soc_max) into all_active_keep/all_active_keep_max.
@@ -970,6 +994,7 @@ def fetch_sensor_data(self, save=True):
         self.pv_forecast_minute = {}
         self.pv_forecast_minute10 = {}
         self.pv_forecast_minute90 = {}
+        self.pv_light_dark = {}
         # See Plan.refresh_pv_forecast_minute90(): both series are re-fetched together below, so no
         # earlier pair of signatures may be held against them
         self.pv_forecast_minute90_signatures = None
@@ -1251,6 +1276,12 @@ def fetch_sensor_data(self, save=True):
         if self.rate_import or self.rate_export:
             self.set_rate_thresholds()
 
+        # Needed here, ahead of "Find charging windows" below, because calc_pv_light_dark() reads
+        # self.pv_forecast_minute to locate dawn - it was previously fetched further down in this
+        # function, after the dawn calculation had already run against the freshly-reset empty dict
+        # from the top of fetch_sensor_data(), so the dawn split could never actually trigger (#4699).
+        pv_light_dark = self.fetch_pv_forecast_and_dawn()
+
         # Find discharging windows
         if self.rate_export:
             self.high_export_rates, lowest, highest = self.rate_scan_window(self.rate_export, 5, self.rate_export_cost_threshold, True, alt_rates=self.rate_import)
@@ -1261,8 +1292,6 @@ def fetch_sensor_data(self, save=True):
 
         # Find charging windows
         if self.rate_import:
-            pv_light_dark = self.calc_pv_light_dark()
-
             # Find charging window
             self.low_rates, lowest, highest = self.rate_scan_window(self.rate_import, 5, self.rate_import_cost_threshold, False, alt_rates=self.rate_export, pv_light_dark=pv_light_dark)
             self.log("Low Import rate found rates in range {}{} to {}{}".format(lowest, curr, highest, curr))
@@ -1299,9 +1328,6 @@ def fetch_sensor_data(self, save=True):
         if self.import_today:
             self.cost_today_sofar, self.carbon_today_sofar = self.today_cost(self.import_today, self.export_today, self.car_charging_energy, self.load_minutes, save=save)
 
-        # Fetch PV forecast if enabled, today must be enabled, other days are optional
-        self.pv_forecast_minute, self.pv_forecast_minute10, self.pv_forecast_minute90 = self.fetch_pv_forecast()
-
         if self.load_minutes and not self.load_forecast_only and not self.load_forecast_history:
             # Apply modal filter to historical data. Skipped for days_previous_auto: the weighted-bucket
             # forecast deliberately ignores missing-data buckets, so the gap-filling/padding the modal filter
@@ -1886,13 +1912,21 @@ def calc_dawn(self):
         day correctly stay all-dark (dawn never crosses) and a polar-day day correctly stay all-light
         (crossed from the very first bucket).
 
-        The threshold itself is LOW_POWER_PV_LIGHT_FRACTION of the peak PV forecast anywhere in
-        self.pv_forecast_minute, not a fixed Watts figure, so it scales with the site rather than being
-        picked for a "typical" system size.
-
-        Built from whatever PV forecast is already in self.pv_forecast_minute, which at the point this
-        is called from fetch_sensor_data is up to one cycle stale (refreshed later this same loop by
-        fetch_pv_forecast()) - fine for a forecast that doesn't meaningfully change minute to minute.
+        The threshold is the user-configurable low_power_pv_threshold_w, an absolute Watts figure,
+        rather than a fraction of this forecast's own peak. A fraction-of-peak threshold sounds like it
+        "scales with the site", but it actually scales with today's weather: a heavily overcast day has
+        a low peak of its own, so a fixed fraction of it can call a trickle of PV "light" at a much
+        lower absolute wattage than the same fraction would on a clear day - and there is nothing in
+        Predbat today recording the site's real panel capacity to normalise against instead (only
+        inverter_limit, which caps the inverter's own output, not the array's rating). An absolute
+        figure the user sets once for their own system does not have that problem (#4699 follow-up).
+        The same value and the same comparison is reused by find_charge_rate's low-power abandon check
+        (utils.py) - one threshold answering "is this bright enough to matter" everywhere it is asked,
+        rather than two independently-tuned ones that could disagree on genuine twilight PV.
+
+        Built from self.pv_forecast_minute as populated earlier this same fetch_sensor_data() call by
+        fetch_pv_forecast() - see #4699, where calc_pv_light_dark() ran before that populating call and
+        so always saw the freshly-reset empty dict, silently disabling the dawn split for everyone.
 
         Logs the calculated dawn time (today's, or the earliest day the forecast reaches if today's PV
         data isn't there) each time it runs, so it's visible whether the detected dawn looks sane.
@@ -1901,11 +1935,7 @@ def calc_dawn(self):
         if not self.pv_forecast_minute:
             return pv_light_dark
 
-        peak_pv = max(self.pv_forecast_minute.values())
-        if peak_pv <= 0:
-            return {pv_minute: 0 for pv_minute in self.pv_forecast_minute}
-
-        light_threshold = peak_pv * LOW_POWER_PV_LIGHT_FRACTION
+        light_threshold = self.low_power_pv_threshold_w / MINUTE_WATT
         interval = self.plan_interval_minutes
         bucket_sums = {}
         bucket_counts = {}
@@ -3123,6 +3153,8 @@ def fetch_config_options(self):
         self.set_charge_low_power = self.get_arg("set_charge_low_power")
         self.set_export_low_power = self.get_arg("set_export_low_power")
         self.charge_low_power_margin = self.get_arg("charge_low_power_margin")
+        self.low_power_pv_threshold_w = self.get_arg("low_power_pv_threshold_w")
+        self.set_charge_low_power_solar_full_rate = self.get_arg("set_charge_low_power_solar_full_rate")
         self.calculate_export_first = True
 
         self.set_status_notify = self.get_arg("set_status_notify")

apps/predbat/output.py

@@ -1082,6 +1082,8 @@ def get_charge_rate_kw(self, charge_window_n, minute_start, minute_relative_star
             self.battery_temperature,
             self.battery_temperature_charge_curve,
             pv_window_kwh=pv_window_kwh,
+            low_power_pv_threshold_w=self.low_power_pv_threshold_w,
+            solar_full_rate=self.set_charge_low_power_solar_full_rate,
         )
         return dp2(charge_rate_now_curve * MINUTE_WATT / 1000.0)
 

apps/predbat/predbat.py

@@ -586,6 +586,7 @@ def reset(self):
         self.pv_forecast_minute = {}
         self.pv_forecast_minute10 = {}
         self.pv_forecast_minute90 = {}
+        self.pv_light_dark = {}
         # (p50, p90) content signatures from the previous plan run, used to spot a p90 that has been
         # left behind by a p50 reassigned underneath it - see Plan.refresh_pv_forecast_minute90()
         self.pv_forecast_minute90_signatures = None

apps/predbat/prediction.py

@@ -105,6 +105,8 @@ def __init__(
             self.set_export_window = base.set_export_window
             self.calculate_export_on_pv = base.calculate_export_on_pv
             self.charge_low_power_margin = base.charge_low_power_margin
+            self.low_power_pv_threshold_w = base.low_power_pv_threshold_w
+            self.set_charge_low_power_solar_full_rate = base.set_charge_low_power_solar_full_rate
             self.car_charging_slots = base.car_charging_slots
             # Model-facing car charge limit (#4967): fetch raises this above the real limit for cars
             # following an Octopus Intelligent dispatch plan with consider_full off, making the fill
@@ -1054,6 +1056,8 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
                     battery_temperature,
                     self.battery_temperature_charge_curve,
                     pv_window_kwh=pv_window_kwh,
+                    low_power_pv_threshold_w=self.low_power_pv_threshold_w,
+                    solar_full_rate=self.set_charge_low_power_solar_full_rate,
                 )
                 charge_rate_now_curve_step = charge_rate_now_curve * step
 

apps/predbat/tests/test_find_charge_rate.py

@@ -74,6 +74,10 @@ def test_find_charge_rate_pv_overlap(my_predbat):
 
     Throttling the charge rate while the sun is shining stops PV going into the battery, the surplus
     is exported cheaply and the target is then made up with grid import, which increases the plan cost.
+
+    set_charge_low_power_solar_full_rate turns that bypass off for a window where the trade does not
+    hold - a free or very cheap import period, where the spilled PV costs nothing to buy back and the
+    throttled rate is wanted anyway (#4975).
     """
     failed = 0
 
@@ -104,7 +108,7 @@ def test_find_charge_rate_pv_overlap(my_predbat):
         0.96,
         log_to,
     ]
-    kwargs = {"battery_temperature": 17.0, "battery_temperature_curve": battery_temperature_curve, "current_charge_rate": max_rate / MINUTE_WATT}
+    kwargs = {"battery_temperature": 17.0, "battery_temperature_curve": battery_temperature_curve, "current_charge_rate": max_rate / MINUTE_WATT, "low_power_pv_threshold_w": my_predbat.low_power_pv_threshold_w}
 
     # Without PV low power mode should slow the charge down below the max rate
     dark_rate, dark_rate_real = find_charge_rate(*args, **kwargs)
@@ -127,6 +131,23 @@ def test_find_charge_rate_pv_overlap(my_predbat):
         print("**** ERROR: PV production in the window should force the max rate {}W, got {}W ****".format(max_rate, sun_rate * MINUTE_WATT))
         failed = 1
 
+    # With the full rate in solar switch off, the same PV must not force the max rate any more - the
+    # window is throttled on its own charge_left / minutes_left just as a dark one would be (#4975)
+    kwargs_no_full_rate = dict(kwargs)
+    kwargs_no_full_rate["solar_full_rate"] = False
+    opt_out_rate, opt_out_rate_real = find_charge_rate(*args, pv_window_kwh=2.0, **kwargs_no_full_rate)
+    print("With PV, full rate in solar off - Best_rate {} Best_rate_real {}".format(opt_out_rate * MINUTE_WATT, opt_out_rate_real * MINUTE_WATT))
+    if opt_out_rate != dark_rate:
+        print("**** ERROR: with solar_full_rate off the PV should not change the rate, expected {}W got {}W ****".format(dark_rate * MINUTE_WATT, opt_out_rate * MINUTE_WATT))
+        failed = 1
+
+    # The switch only has meaning when there is PV to abandon low power for - with no PV in the window
+    # it must make no difference either way
+    no_pv_opt_out_rate, _ = find_charge_rate(*args, **kwargs_no_full_rate)
+    if no_pv_opt_out_rate != dark_rate:
+        print("**** ERROR: with no PV solar_full_rate should not change the rate, expected {}W got {}W ****".format(dark_rate * MINUTE_WATT, no_pv_opt_out_rate * MINUTE_WATT))
+        failed = 1
+
     return failed