Stamp day_of_week rules across the whole rate horizon, not just day+2 (#5168 review)

basic_rates() now runs its per-day loop out to the end of the horizon rate_replicate()
covers, instead of modelling two days and seeding a third with a separate write. Every day
is filtered against its own weekday, so days past the third no longer copy the previous
day's pattern, a midnight-spanning rule's early hours land on the right day, and
rate_increment and load_scaling behave the same on every day. The weekday offset uses
floor division, so yesterday's minutes are checked against yesterday's weekday.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: CI

apps/predbat/fetch.py

@@ -2123,8 +2123,9 @@ def apply_manual_rates(self, rates, manual_items, is_import=True, rate_replicate
 
     def basic_rates(self, info, rtype, prev=None, rate_replicate=None, include_manual_api=True):
         """
-        Work out the energy rates based on user supplied time periods
-        works on a 24-hour period only and then gets replicated later for future days
+        Work out the energy rates based on user supplied time periods. Weekly rules on the base
+        tariff are stamped across the whole horizon, each day against its own day_of_week; any
+        minute no rule claims is left for rate_replicate() to fill later
 
         include_manual_api should be False for callers (e.g. tariff comparison, annual replay)
         that simulate a tariff other than the live one - the live system's manual API overrides
@@ -2286,16 +2287,28 @@ def basic_rates(self, info, rtype, prev=None, rate_replicate=None, include_manua
 
                 day_of_week_midnight = self.midnight_utc.weekday()
 
+                # A weekly rule on the base tariff is stamped over the whole horizon rate_replicate()
+                # goes on to cover, not just the days modelled above, so every day is filtered against
+                # its own weekday. rate_replicate() fills a missing minute from the same time of day 24
+                # hours earlier and knows nothing about day_of_week, so a day left to it takes the
+                # previous day's pattern - a weekend rule flattening the weekday peaks (batpred#5168).
+                # Overrides (prev) already span the replicated horizon via max_minute.
+                if not date and not prev:
+                    stamp_end = max(max_minute + 24 * 60, self.forecast_minutes + 48 * 60)
+                else:
+                    stamp_end = max_minute
+
                 # Store rates against range
                 if end_minutes >= (-48 * 60) and start_minutes < max_minute:
                     for minute in range(start_minutes, end_minutes):
                         minute_mod = minute % max_minute
                         if (not date) or (minute >= (-24 * 60) and minute < max_minute):
                             minute_index = minute_mod - 24 * 60
                             # For incremental adjustments we have to loop over 24-hour periods
-                            while minute_index < max_minute:
+                            while minute_index < stamp_end:
                                 if not date or (minute_index >= start_minutes and minute_index < end_minutes):
-                                    current_day_of_week = (day_of_week_midnight + int(minute_index / (24 * 60))) % 7
+                                    # Floor division, so yesterday's minutes are checked against yesterday's weekday
+                                    current_day_of_week = (day_of_week_midnight + minute_index // (24 * 60)) % 7
                                     if not day_of_week or (current_day_of_week in day_of_week):
                                         if rate_increment:
                                             rates[minute_index] = rates.get(minute_index, 0.0) + rate
@@ -2308,20 +2321,6 @@ def basic_rates(self, info, rtype, prev=None, rate_replicate=None, include_manua
                                         if date:
                                             break
                                 minute_index += 24 * 60
-                            if not date and not prev:
-                                # Seed the slot past the days modelled above so rate_replicate() has
-                                # something to copy forward. This has to honour the rule's day filter
-                                # in the same way the loop just did, or the last rule processed for a
-                                # given time of day wins out there whatever the weekday - a weekend
-                                # rule flattening the weekday peaks two days out (batpred#5168).
-                                # Minutes left unwritten here are no loss: rate_replicate() fills them
-                                # from the same time of day 24 hours earlier.
-                                extend_minute = minute_mod + max_minute
-                                extend_day_of_week = (day_of_week_midnight + int(extend_minute / (24 * 60))) % 7
-                                if not day_of_week or (extend_day_of_week in day_of_week):
-                                    rates[extend_minute] = rate
-                                    if load_scaling is not None:
-                                        self.load_scaling_dynamic[extend_minute] = load_scaling
             else:
                 self.log("Warn: Bad rate data provided in energy rates type {} {}".format(rtype, this_rate))
 

apps/predbat/tests/test_basic_rates.py

@@ -117,9 +117,8 @@ def test_basic_rates(my_predbat):
     my_predbat.midnight_utc = my_predbat.local_tz.localize(datetime.strptime("2025-07-05T00:00:00", "%Y-%m-%dT%H:%M:%S"))  # Saturday (day 6)
     old_minutes_now = my_predbat.minutes_now
     my_predbat.minutes_now = 14 * 60  # 14:00 Saturday afternoon
-    # Tests 6 and 7 also assert the third day, which basic_rates() leaves to rate_replicate() where
-    # no rule seeds it - pin the forecast length so that day is replicated, and the future rate
-    # offset so the replicated values are comparable against the tariff's own rates
+    # Tests 6 and 7 also assert the third day - pin the forecast length so it lies inside the horizon,
+    # and the future rate offset so any replicated minute is comparable against the tariff's own rates
     old_forecast_minutes = my_predbat.forecast_minutes
     old_rate_offset_import = my_predbat.metric_future_rate_offset_import
     my_predbat.forecast_minutes = 24 * 60
@@ -139,11 +138,8 @@ def test_basic_rates(my_predbat):
     failed |= assert_rates(results, 24 * 60, 48 * 60, 8)  # All day Sunday = 8
 
     # Day 3 (Monday): the weekday pattern must survive 48 hours out (issue #5168). basic_rates()
-    # seeds the slot past the days it models so rate_replicate() has something to copy forward,
-    # and that seeding write used to ignore day_of_week - so the weekend rule, being processed
-    # last, flattened Monday to the weekend rate and erased the weekday peak. 00:00-07:00 is not
-    # seeded by any rule here (the off-peak window seeds the fourth day from 22:00 onwards), so it
-    # comes from rate_replicate() copying the same time of day 24 hours earlier
+    # used to seed that day with a write that ignored day_of_week, so the weekend rule, being
+    # processed last, flattened Monday to the weekend rate and erased the weekday peak
     failed |= assert_rates(results, 48 * 60, 48 * 60 + 7 * 60, 8)  # 00:00-07:00 off-peak
     failed |= assert_rates(results, 48 * 60 + 7 * 60, 48 * 60 + 22 * 60, 14)  # 07:00-22:00 peak
     failed |= assert_rates(results, 48 * 60 + 22 * 60, 72 * 60, 8)  # 22:00-24:00 off-peak
@@ -293,10 +289,10 @@ def test_basic_rates(my_predbat):
         my_predbat.manual_api = old_manual_api
         my_predbat.had_errors = old_had_errors
 
-    # Test 13: the export config from issue #5168 - weekday rules tiling the whole day plus a
-    # weekend catch-all. The weekend rule is processed last, and before the fix its seeding write
-    # for the day beyond the ones basic_rates() models ignored day_of_week, so it flattened that
-    # day to the weekend rate and the weekday peaks vanished 48 hours out
+    # Test 13: the export config from issue #5168 - weekday rules covering the day plus a weekend
+    # catch-all. The weekend rule is processed last, and before the fix its seeding write for the
+    # day beyond the ones basic_rates() modelled ignored day_of_week, so it flattened that day to
+    # the weekend rate and the weekday peaks vanished 48 hours out
     print("*** Running test: Weekend day_of_week rule must not flatten future weekdays (issue #5168)")
     saved_midnight_utc = my_predbat.midnight_utc
     my_predbat.midnight_utc = my_predbat.local_tz.localize(datetime.strptime("2025-07-06T00:00:00", "%Y-%m-%dT%H:%M:%S"))  # Sunday, so the third day is Tuesday
@@ -312,9 +308,9 @@ def test_basic_rates(my_predbat):
         results = my_predbat.basic_rates(weekend_catch_all, "export")
         results, results_replicated = my_predbat.rate_replicate(results, is_import=False, is_gas=False)
 
-        # Day 3 (Tuesday) must keep the weekday peaks. The rules end on :59:59 so each leaves the
-        # final minute of its window unset - assert the interiors, which are the ones the rules
-        # actually write
+        # Day 3 (Tuesday) must keep the weekday peaks. The rules end on :59:59, which floors to the
+        # :59 minute, and the window excludes its end minute - so each rule leaves its final minute
+        # (06:59, 08:59, ...) to rate_replicate(). Assert the interiors, which the rules write
         failed |= assert_rates(results, 48 * 60, 48 * 60 + 6 * 60 + 59, 11.5)  # 00:00-07:00 off-peak
         failed |= assert_rates(results, 48 * 60 + 7 * 60, 48 * 60 + 8 * 60 + 59, 23.0)  # 07:00-09:00 peak
         failed |= assert_rates(results, 48 * 60 + 9 * 60, 48 * 60 + 16 * 60 + 59, 11.5)  # 09:00-17:00 off-peak
@@ -323,4 +319,72 @@ def test_basic_rates(my_predbat):
     finally:
         my_predbat.midnight_utc = saved_midnight_utc
 
+    # Test 14: every day out to the end of the horizon rate_replicate() covers keeps its own weekday's
+    # pattern, whichever order the rules are listed in. rate_replicate() fills a missing minute from
+    # the same time of day 24 hours earlier and knows nothing about day_of_week, so any day left to
+    # it takes the previous day's pattern - Monday copying Sunday's flat rate, say. The weekday
+    # off-peak differs from the weekend rate so the early hours of the midnight-spanning window are
+    # checked too, and the peak carries a load_scaling to check that write follows the same days
+    print("*** Running test: day_of_week honoured on every day of the horizon, in either rule order (issue #5168)")
+    saved_midnight_utc = my_predbat.midnight_utc
+    saved_forecast_minutes = my_predbat.forecast_minutes
+    saved_rate_offset_import = my_predbat.metric_future_rate_offset_import
+    saved_load_scaling_dynamic = my_predbat.load_scaling_dynamic
+    # rate_replicate() then runs to the end of the fifth day, and replicated minutes stay comparable
+    my_predbat.forecast_minutes = 72 * 60
+    my_predbat.metric_future_rate_offset_import = 0
+    try:
+        weekday_rules = [
+            {"start": "07:00:00", "end": "22:00:00", "rate": 14.0, "day_of_week": "1,2,3,4,5", "load_scaling": 1.5},
+            {"start": "22:00:00", "end": "07:00:00", "rate": 12.0, "day_of_week": "1,2,3,4,5"},
+        ]
+        weekend_rule = [{"rate": 8.0, "day_of_week": "6,7"}]
+        for midnight in ("2025-07-04", "2025-07-05"):  # Friday and Saturday
+            for rules in (weekday_rules + weekend_rule, weekend_rule + weekday_rules):
+                my_predbat.midnight_utc = my_predbat.local_tz.localize(datetime.strptime(midnight, "%Y-%m-%d"))
+                my_predbat.load_scaling_dynamic = {}
+                results = my_predbat.basic_rates(rules, "import")
+                results, results_replicated = my_predbat.rate_replicate(results, is_import=True, is_gas=False)
+                for day in range(5):
+                    day_start = day * 24 * 60
+                    if (my_predbat.midnight_utc.weekday() + day) % 7 >= 5:
+                        failed |= assert_rates(results, day_start, day_start + 24 * 60, 8.0)
+                        expect_scaling = None
+                    else:
+                        failed |= assert_rates(results, day_start, day_start + 7 * 60, 12.0)
+                        failed |= assert_rates(results, day_start + 7 * 60, day_start + 22 * 60, 14.0)
+                        failed |= assert_rates(results, day_start + 22 * 60, day_start + 24 * 60, 12.0)
+                        expect_scaling = 1.5
+                    scaling = my_predbat.load_scaling_dynamic.get(day_start + 12 * 60)
+                    if scaling != expect_scaling:
+                        print("ERROR: load_scaling at noon on day {} from midnight {} should be {} got {}".format(day, midnight, expect_scaling, scaling))
+                        failed = 1
+    finally:
+        my_predbat.midnight_utc = saved_midnight_utc
+        my_predbat.forecast_minutes = saved_forecast_minutes
+        my_predbat.metric_future_rate_offset_import = saved_rate_offset_import
+        my_predbat.load_scaling_dynamic = saved_load_scaling_dynamic
+
+    # Test 15: yesterday's minutes are filtered against yesterday's weekday. The day offset was
+    # int(minute / 1440), which truncates towards zero, so every minute of yesterday bar its first
+    # was checked against today's weekday instead
+    print("*** Running test: yesterday's rates follow yesterday's day_of_week")
+    saved_midnight_utc = my_predbat.midnight_utc
+    my_predbat.midnight_utc = my_predbat.local_tz.localize(datetime.strptime("2025-07-07T00:00:00", "%Y-%m-%dT%H:%M:%S"))  # Monday, so yesterday is Sunday
+    try:
+        results = my_predbat.basic_rates([{"rate": 14.0, "day_of_week": "1,2,3,4,5"}, {"rate": 8.0, "day_of_week": "6,7"}], "import")
+        failed |= assert_rates(results, -24 * 60, 0, 8.0)
+        failed |= assert_rates(results, 0, 24 * 60, 14.0)
+    finally:
+        my_predbat.midnight_utc = saved_midnight_utc
+
+    # Test 16: a rate_increment rule adds to the rate already there on every day it is stamped on,
+    # including the days past the two modelled explicitly - the seeding write for the third day
+    # used to assign the bare increment there
+    print("*** Running test: rate_increment adds to the base rate on every day")
+    results = my_predbat.basic_rates([{"rate": 10.0}, {"start": "17:00:00", "end": "19:00:00", "rate_increment": 5.0}], "import")
+    for day in range(3):
+        failed |= assert_rates(results, day * 24 * 60 + 17 * 60, day * 24 * 60 + 19 * 60, 15.0)
+        failed |= assert_rates(results, day * 24 * 60 + 19 * 60, day * 24 * 60 + 20 * 60, 10.0)
+
     return failed