Fix fetch crash

Support day/night tariff

Author: springfall2008

apps/predbat/fetch.py

@@ -758,13 +758,14 @@ def minute_data(
                     if minute in mdata:
                         last_sample_minute = minute
                         break
-                last_sample_value = mdata[last_sample_minute]
-                last_but_one_minute_sample = mdata[last_sample_minute + 1] if (last_sample_minute + 1) in mdata else last_sample_value
-                step = last_sample_value - last_but_one_minute_sample
-                if last_sample_minute < 10 * 60 and last_sample_minute > 0:
-                    for minute in range(last_sample_minute):
-                        if minute not in mdata:
-                            mdata[minute] = dp4(newest_state + step * (last_sample_minute - minute))
+                if last_sample_minute > 0:
+                    last_sample_value = mdata[last_sample_minute]
+                    last_but_one_minute_sample = mdata[last_sample_minute + 1] if (last_sample_minute + 1) in mdata else last_sample_value
+                    step = last_sample_value - last_but_one_minute_sample
+                    if last_sample_minute < 10 * 60 and last_sample_minute > 0:
+                        for minute in range(last_sample_minute):
+                            if minute not in mdata:
+                                mdata[minute] = dp4(newest_state + step * (last_sample_minute - minute))
 
             # Fill from last sample until now
             for minute in range(60 * 24 * days):

apps/predbat/octopus.py

@@ -26,6 +26,16 @@
 DATE_STR_FORMAT = "%Y-%m-%d"
 DATE_TIME_STR_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
 
+# Hard-wired smart meter day/night rate times
+OCTOPUS_NIGHT_RATE_START_HOUR = 0
+OCTOPUS_NIGHT_RATE_START_MINUTE = 30
+OCTOPUS_NIGHT_RATE_END_HOUR = 7
+OCTOPUS_NIGHT_RATE_END_MINUTE = 30
+
+OCTOPUS_DAY_RATE_START_HOUR = 7
+OCTOPUS_DAY_RATE_START_MINUTE = 30
+OCTOPUS_DAY_RATE_END_HOUR = 0
+OCTOPUS_DAY_RATE_END_MINUTE = 30
 
 def is_active(now_utc, activeFrom, activeTo):
     if not activeFrom:
@@ -673,6 +683,50 @@ async def async_get_saving_sessions(self, account_id):
         else:
             return response_data.get("savingSessions", {})
 
+    async def async_get_day_night_rates(self, url):
+        """
+        Get day and night rates from Octopus
+        """
+        mdata = []
+        self.log("Info: Octopus tariff has day and night rates, fetching both")
+        url_day = url.replace("standard-unit-rates", "day-unit-rates")
+        url_night = url.replace("standard-unit-rates", "night-unit-rates")
+        result_day = await self.fetch_url_cached(url_day)
+        result_night = await self.fetch_url_cached(url_night)
+        # is_night_rate = self.__is_between_times(rate, "00:30:00", "07:30:00", True)
+        # Find the current day rate by scanning all the values looking at valid from date and picking the latest that is before now
+        current_day_rate = None
+        current_night_rate = None
+        for rate in result_day:
+            valid_from_stamp = rate.get("valid_from", "")
+            # Convert from string to datetime
+            valid_from_stamp = datetime.strptime(valid_from_stamp, DATE_TIME_STR_FORMAT)
+            if valid_from_stamp <= self.now_utc:
+                current_day_rate = rate.get('value_inc_vat', None)
+        for rate in result_night:
+            valid_from_stamp = rate.get("valid_from", "")
+            # Convert from string to datetime
+            valid_from_stamp = datetime.strptime(valid_from_stamp, DATE_TIME_STR_FORMAT)
+            if valid_from_stamp <= self.now_utc:
+                current_night_rate = rate.get('value_inc_vat', None)
+        self.log("Info: Current day rate {} night rate {}".format(current_day_rate, current_night_rate))
+        if current_day_rate is not None and current_night_rate is not None:
+            # Now create a combined list of rates, start from 2 days back and go forward 3 days with the day and night rates
+            night_start_time = self.now_utc.replace(hour=OCTOPUS_NIGHT_RATE_START_HOUR, minute=OCTOPUS_NIGHT_RATE_START_MINUTE, second=0, microsecond=0) - timedelta(days=2)
+            night_end_time = night_start_time.replace(hour=OCTOPUS_NIGHT_RATE_END_HOUR, minute=OCTOPUS_NIGHT_RATE_END_MINUTE)
+            day_start_time = night_start_time.replace(hour=OCTOPUS_DAY_RATE_START_HOUR, minute=OCTOPUS_DAY_RATE_START_MINUTE)
+            day_end_time = night_start_time.replace(hour=OCTOPUS_DAY_RATE_END_HOUR, minute=OCTOPUS_DAY_RATE_END_MINUTE) + timedelta(days=1)
+            for day in range(8):
+                # Night rate
+                mdata.append({"valid_from": night_start_time.strftime(DATE_TIME_STR_FORMAT), "valid_to": night_end_time.strftime(DATE_TIME_STR_FORMAT), "value_inc_vat": current_night_rate})
+                # Day rate
+                mdata.append({"valid_from": day_start_time.strftime(DATE_TIME_STR_FORMAT), "valid_to": day_end_time.strftime(DATE_TIME_STR_FORMAT), "value_inc_vat": current_day_rate})
+                night_start_time += timedelta(days=1)
+                night_end_time += timedelta(days=1)
+                day_start_time += timedelta(days=1)
+                day_end_time += timedelta(days=1)  
+        return mdata
+          
     async def async_download_octopus_url(self, url):
         """
         Download octopus rates directly from a URL
@@ -683,7 +737,7 @@ async def async_download_octopus_url(self, url):
         while url and pages < 3:
             self.requests_total += 1
             r = requests.get(url)
-            if r.status_code not in [200, 201]:
+            if r.status_code not in [200, 201, 400]:
                 self.failures_total += 1
                 self.log("Warn: Error downloading Octopus data from URL {}, code {}".format(url, r.status_code))
                 return {}
@@ -694,9 +748,28 @@ async def async_download_octopus_url(self, url):
                 self.failures_total += 1
                 self.log("Warn: Error downloading Octopus data from URL {} (JSONDecodeError)".format(url))
                 return {}
+            
+            if r.status_code == 400:
+                detail = data.get("detail", "")
+                if 'This tariff has day and night rates' in detail:
+                    self.log("Info: Octopus tariff has day and night rates, fetching both")
+                    mdata = await self.async_get_day_night_rates(url)
+                    if mdata:
+                        return mdata
+                    else:
+                        self.failures_total += 1
+                        self.log("Warn: Error downloading Octopus data from URL {} (No Results)".format(url))
+                        return {}
+                else:
+                    self.failures_total += 1
+                    self.log("Warn: Error downloading Octopus data from URL {} (400) - {}".format(url, detail))
+                    return {}
+
             if "results" in data:
                 mdata += data["results"]
             else:
+                detail = data.get("detail", "")
+
                 self.failures_total += 1
                 self.log("Warn: Error downloading Octopus data from URL {} (No Results)".format(url))
                 return {}
@@ -751,7 +824,7 @@ async def fetch_tariffs(self, tariffs):
                 tariff_type = "gas"
             else:
                 tariff_type = "electricity"
-            tariffs[tariff]["data"] = await self.fetch_url_cached(f"https://api.octopus.energy/v1/products/{product_code}/{tariff_type}-tariffs/{tariff_code}/standard-unit-rates/")
+            tariffs[tariff]["data"] = await self.fetch_url_cached(f"https://api.octopus.energy/v1/products/{product_code}/{tariff_type}-tariffs/{tariff_code}/standard-unit-rates/")            
             tariffs[tariff]["standing"] = await self.fetch_url_cached(f"https://api.octopus.energy/v1/products/{product_code}/{tariff_type}-tariffs/{tariff_code}/standing-charges/")
 
             entity_id = "sensor.predbat_octopus_" + self.account_id.replace("-", "_") + "_" + tariff

apps/predbat/unit_test.py

@@ -15,6 +15,7 @@
 import glob
 from datetime import datetime, timedelta
 import argparse
+import asyncio
 
 try:
     import requests
@@ -9133,6 +9134,48 @@ def run_test_units(my_predbat):
     return failed
 
 
+def test_download_octopus_url_wrapper(my_predbat):
+    """
+    Wrapper to run the async test function
+    """
+    return asyncio.run(test_download_octopus_url(my_predbat))
+
+async def test_download_octopus_url(my_predbat):
+    """
+    Test the download_octopus_url function
+    """
+    print("**** Running download_octopus_url tests ****")
+    failed = False
+    
+    # Test URL for VAR-22-11-01 tariff
+    test_url = "https://api.octopus.energy/v1/products/VAR-22-11-01/electricity-tariffs/E-2R-VAR-22-11-01-A/standard-unit-rates/"
+    
+    # Test the download function
+    api = OctopusAPI("", "", my_predbat)
+    api.now_utc = my_predbat.now_utc
+    rates_data = await api.async_download_octopus_url(test_url)
+
+    # Basic validation checks
+    if not rates_data:
+        print("ERROR: No rate data downloaded from URL {}".format(test_url))
+        failed = True
+    else:
+        print("Successfully downloaded {} rate points from VAR-22-11-01 tariff".format(len(rates_data)))
+        pdata = my_predbat.minute_data(rates_data, my_predbat.forecast_days + 1, my_predbat.midnight_utc, "value_inc_vat", "valid_from", backwards=False, to_key="valid_to")
+        if len(pdata) < 24 * 60:
+            print("ERROR: Expecting at least {} minutes of rate data got {}".format(24 * 60, len(pdata)))
+            failed = True
+        else:
+            print("Successfully processed {} minutes of rate data".format(len(pdata)))
+            night_rate = pdata.get(60)
+            day_rate = pdata.get(600)
+            if night_rate == day_rate:
+                print("ERROR: Expecting different night and day rates got {} and {}".format(night_rate, day_rate))
+                failed = True
+
+    return failed
+
+
 def main():
     # Parse command line arguments
     parser = argparse.ArgumentParser(description="Predbat unit tests")
@@ -9179,6 +9222,8 @@ def main():
         if args.perf_only:
             return failed
 
+    if not failed:
+        failed |= test_download_octopus_url_wrapper(my_predbat)
     if not failed:
         failed |= test_plugin_startup_order(my_predbat)