Dynamic load feature (#2702)

* Dynamic load feature

* [pre-commit.ci lite] apply automatic fixes

* Fixes

* Fixes to ge cloud timeout
fixes to logfile search on web
fixes to code for dynamic load

* [pre-commit.ci lite] apply automatic fixes

---------

Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>

Author: springfall2008

apps/predbat/config.py

@@ -504,6 +504,12 @@
         "name": "metric_pv_calibration_enable",
         "friendly_name": "Enable use of Calibrated PV data",
         "type": "switch",
+        "default": True,
+    },
+    {
+        "name": "metric_dynamic_load_adjust",
+        "friendly_name": "Dynamic Load Adjust",
+        "type": "switch",
         "default": False,
     },
     {

apps/predbat/energydataservice.py

@@ -34,7 +34,6 @@ def fetch_energidataservice_rates(self, entity_id, adjust_key=None):
         if data_all:
             # Sort to be safe
             data_all.sort(key=lambda e: self._parse_iso(e.get("hour")) or datetime.min)
-            print(data_all)
 
             # Add tariffs (HH:MM → H → HH → raw ISO)
             for entry in data_all:
@@ -53,7 +52,6 @@ def fetch_energidataservice_rates(self, entity_id, adjust_key=None):
                 scale=1.0,
                 use_cent=use_cent,
             )
-            print(rate_data)
 
         return rate_data
 

apps/predbat/fetch.py

@@ -87,6 +87,7 @@ def step_data_history(
         base_offset=None,
         flip=False,
         load_adjust={},
+        load_baseline={},
     ):
         """
         Create cached step data for historical array
@@ -132,6 +133,10 @@ def step_data_history(
             load_extra = max(load_extra, -value)  # Don't allow going to negative load values
             values[minute] = dp4((value + load_extra) * scaling_dynamic * scale_today * scale_fixed)
 
+            # Apply dynamic baseline
+            if minute_absolute in load_baseline:
+                values[minute] = max(values[minute], load_baseline[minute_absolute])
+
         # Simple divergence model keeps the same total but brings PV/Load up and down every 5 minutes
         if cloud_factor and cloud_factor > 0:
             for minute in range(0, self.forecast_minutes, step):
@@ -954,11 +959,13 @@ def fetch_sensor_data(self):
                 self.load_minutes, self.load_minutes_age = self.minute_data_load(self.now_utc, "load_today", self.max_days_previous, required_unit="kWh", load_scaling=self.load_scaling)
                 self.log("Found {} load_today datapoints going back {} days".format(len(self.load_minutes), self.load_minutes_age))
                 self.load_minutes_now = max(self.load_minutes.get(0, 0) - self.load_minutes.get(self.minutes_now, 0), 0)
+                self.load_last_period = (self.load_minutes.get(0, 0) - self.load_minutes.get(PREDICT_STEP, 0)) * 60 / PREDICT_STEP
             else:
                 if self.load_forecast:
                     self.log("Using load forecast from load_forecast sensor")
                     self.load_minutes_now = self.load_forecast.get(0, 0)
                     self.load_minutes_age = 0
+                    self.load_last_period = 0
                 else:
                     self.log("Error: You have not set load_today or load_forecast, you will have no load data")
                     self.record_status(message="Error: load_today not set correctly", had_errors=True)
@@ -2063,6 +2070,7 @@ def fetch_config_options(self):
         self.metric_future_rate_offset_export = self.get_arg("metric_future_rate_offset_export")
         self.metric_inday_adjust_damping = self.get_arg("metric_inday_adjust_damping")
         self.metric_pv_calibration_enable = self.get_arg("metric_pv_calibration_enable")
+        self.metric_dynamic_load_adjust = self.get_arg("metric_dynamic_load_adjust")
         self.rate_low_threshold = self.get_arg("rate_low_threshold")
         self.rate_high_threshold = self.get_arg("rate_high_threshold")
         self.inverter_soc_reset = self.get_arg("inverter_soc_reset")

apps/predbat/gecloud.py

@@ -6,10 +6,11 @@
 
 import requests
 from datetime import timedelta, datetime, timezone
-from utils import str2time, dp1
+from utils import str2time, dp1, dp2
 import asyncio
 import random
 import time
+from config import PREDICT_STEP
 
 """
 GE Cloud data download
@@ -223,7 +224,7 @@ def wait_api_started(self):
         """
         self.log("GECloud: Waiting for API to start")
         count = 0
-        while not self.api_started and count < 120:
+        while not self.api_started and count < 240:
             time.sleep(1)
             count += 1
         if not self.api_started:
@@ -1332,9 +1333,9 @@ def download_ge_data(self, now_utc):
             url = "https://api.givenergy.cloud/v1/inverter/{}/data-points/{}".format(geserial, datestr)
             while url:
                 if "?" in url:
-                    url += "&pageSize=4096"
+                    url += "&pageSize=8000"
                 else:
-                    url += "?pageSize=4096"
+                    url += "?pageSize=8000"
                 data = self.get_ge_url(url, headers, now_utc, 30 if days_prev == 0 else 8 * 60)
                 darray = data.get("data", None)
                 if darray is None:
@@ -1386,8 +1387,32 @@ def download_ge_data(self, now_utc):
         self.pv_today = self.minute_data(mdata, self.max_days_previous, now_utc, "pv", "last_updated", backwards=True, smoothing=True, scale=self.import_export_scaling, clean_increment=True)
 
         self.load_minutes_now = self.load_minutes.get(0, 0) - self.load_minutes.get(self.minutes_now, 0)
+        self.load_last_period = (self.load_minutes.get(0, 0) - self.load_minutes.get(PREDICT_STEP, 0)) * 60 / PREDICT_STEP
         self.import_today_now = self.import_today.get(0, 0) - self.import_today.get(self.minutes_now, 0)
         self.export_today_now = self.export_today.get(0, 0) - self.export_today.get(self.minutes_now, 0)
         self.pv_today_now = self.pv_today.get(0, 0) - self.pv_today.get(self.minutes_now, 0)
+
+        self.log("GE GECloud load_last_period is {} kW".format(dp2(self.load_last_period)))
+        # More up to date sensors for current values if set
+        if "load_today" in self.args:
+            load_minutes, load_minutes_age = self.minute_data_load(self.now_utc, "load_today", self.max_days_previous, required_unit="kWh", load_scaling=self.load_scaling)
+            self.load_minutes_now = max(load_minutes.get(0, 0) - load_minutes.get(self.minutes_now, 0), 0)
+            self.load_last_period = (load_minutes.get(0, 0) - load_minutes.get(PREDICT_STEP, 0)) * 60 / PREDICT_STEP
+            self.log("GE GECloud load_last_period from immediate sensor is {} kW".format(dp2(self.load_last_period)))
+
+        if "import_today" in self.args:
+            import_today = self.minute_data_import_export(self.now_utc, "import_today", scale=self.import_export_scaling, required_unit="kWh")
+            self.import_today_now = max(import_today.get(0, 0) - import_today.get(self.minutes_now, 0), 0)
+
+        # Load export today data
+        if "export_today" in self.args:
+            export_today = self.minute_data_import_export(self.now_utc, "export_today", scale=self.import_export_scaling, required_unit="kWh")
+            self.export_today_now = max(export_today.get(0, 0) - export_today.get(self.minutes_now, 0), 0)
+
+        # PV today data
+        if "pv_today" in self.args:
+            pv_today = self.minute_data_import_export(self.now_utc, "pv_today", required_unit="kWh")
+            self.pv_today_now = max(pv_today.get(0, 0) - pv_today.get(self.minutes_now, 0), 0)
+
         self.log("Downloaded {} datapoints from GE going back {} days".format(len(self.load_minutes), self.load_minutes_age))
         return True

apps/predbat/plan.py

@@ -12,8 +12,8 @@
 import traceback
 from datetime import datetime, timedelta
 from multiprocessing import Pool, cpu_count
-from config import PREDICT_STEP, TIME_FORMAT
-from utils import calc_percent_limit, dp0, dp1, dp2, dp3, dp4, remove_intersecting_windows, calc_percent_limit
+from config import PREDICT_STEP, TIME_FORMAT, MINUTE_WATT
+from utils import calc_percent_limit, dp0, dp1, dp2, dp3, dp4, remove_intersecting_windows, calc_percent_limit, in_car_slot
 from prediction import Prediction, wrapped_run_prediction_single, wrapped_run_prediction_charge, wrapped_run_prediction_charge_min_max, wrapped_run_prediction_export, wrapped_run_prediction_charge_min_max
 
 """
@@ -36,6 +36,43 @@ def get(self):
 
 
 class Plan:
+    def dynamic_load(self):
+        # Last period load analysis
+        self.load_last_status = "baseline"
+        if self.load_last_period > self.battery_rate_max_discharge * MINUTE_WATT / 1000:
+            self.load_last_status = "high"
+        elif (self.load_last_period < self.battery_rate_max_discharge * 0.9 * MINUTE_WATT / 1000) and (self.load_last_period < self.car_charging_threshold * 0.9):
+            # Check if the load is less than car charging threshold
+            self.load_last_status = "low"
+        else:
+            self.load_last_status = "baseline"
+
+        self.dynamic_load_baseline = {}
+        if self.metric_dynamic_load_adjust:
+            minutes_now = self.minutes_now
+            minutes_end_slot = int((self.minutes_now + 30) / 30) * 30
+            # When dynamic load is enabled we try can do two things
+            # 1. Increase the load prediction in the current 30 minute period to match the actual load (if its low)
+            # 2. If its low and car charging is predicted then cancel off that car slot
+            if self.load_last_status == "low":
+                for car_n in range(0, self.num_cars):
+                    for slot in self.car_charging_slots[car_n]:
+                        if (slot["start"] < minutes_now) and (slot["start"] < minutes_end_slot) and (slot["end"] > minutes_now):
+                            # If the slot is within the current 30 minute period
+                            self.log("Dynamic load adjust is cancelling car {} slot {}-{} due to low load".format(car_n + 1, slot["start"], slot["end"]))
+                            slot["kwh"] = 0
+            if self.load_last_status == "high":
+                have_printed = False
+                for minute_absolute in range(minutes_now, minutes_end_slot, PREDICT_STEP):
+                    car_load = sum(in_car_slot(minute_absolute, self.num_cars, self.car_charging_slots))
+                    load_last_period = self.load_last_period / 60 * PREDICT_STEP
+                    load_last_period = max(load_last_period - car_load, 0)
+                    if load_last_period > 0:
+                        if not have_printed:
+                            self.log("Dynamic load adjust is setting load minimum {:.2f}kW at {}".format(load_last_period, self.time_abs_str(minute_absolute)))
+                            have_printed = True
+                        self.dynamic_load_baseline[minute_absolute] = load_last_period
+
     def find_price_levels(
         self,
         price_set,
@@ -791,6 +828,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True):
             load_scaling_dynamic=self.load_scaling_dynamic,
             cloud_factor=self.metric_load_divergence,
             load_adjust=self.manual_load_adjust,
+            load_baseline=self.dynamic_load_baseline,
         )
         load_minutes_step10 = self.step_data_history(
             self.load_minutes,
@@ -803,6 +841,7 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True):
             load_scaling_dynamic=self.load_scaling_dynamic,
             cloud_factor=min(self.metric_load_divergence + 0.5, 1.0) if self.metric_load_divergence else None,
             load_adjust=self.manual_load_adjust,
+            load_baseline=self.dynamic_load_baseline,
         )
         pv_forecast_minute_step = self.step_data_history(self.pv_forecast_minute, self.minutes_now, forward=True, cloud_factor=self.metric_cloud_coverage)
         pv_forecast_minute10_step = self.step_data_history(self.pv_forecast_minute10, self.minutes_now, forward=True, cloud_factor=min(self.metric_cloud_coverage + 0.2, 1.0) if self.metric_cloud_coverage else None, flip=True)
@@ -3306,7 +3345,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
                 self.dashboard_item(
                     self.prefix + ".export_energy_h0",
                     state=dp3(export_kwh_h0),
-                    attributes={"friendly_name": "Current export kWh", "state_class": "measurement", "unit_of_measurement": "kWh", "icon": "mdi:transmission-tower-export"},
+                    attributes={"friendly_name": "Current export", "state_class": "measurement", "unit_of_measurement": "kWh", "icon": "mdi:transmission-tower-export"},
                 )
                 self.dashboard_item(
                     self.prefix + ".load_energy",
@@ -3323,7 +3362,12 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
                 self.dashboard_item(
                     self.prefix + ".load_energy_h0",
                     state=dp3(load_kwh_h0),
-                    attributes={"friendly_name": "Current load kWh", "state_class": "measurement", "unit_of_measurement": "kWh", "icon": "mdi:home-lightning-bolt"},
+                    attributes={"friendly_name": "Current load", "state_class": "measurement", "unit_of_measurement": "kWh", "icon": "mdi:home-lightning-bolt"},
+                )
+                self.dashboard_item(
+                    self.prefix + ".load_energy_last_period",
+                    state=dp3(self.load_last_period),
+                    attributes={"friendly_name": "Last period load", "state_class": "measurement", "unit_of_measurement": "kW", "icon": "mdi:home-lightning-bolt", "status": self.load_last_status},
                 )
                 self.dashboard_item(
                     self.prefix + ".pv_energy",
@@ -3333,7 +3377,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
                 self.dashboard_item(
                     self.prefix + ".pv_energy_h0",
                     state=dp3(pv_kwh_h0),
-                    attributes={"friendly_name": "Current PV kWh", "state_class": "measurement", "unit_of_measurement": "kWh", "icon": "mdi:solar-power"},
+                    attributes={"friendly_name": "Current PV", "state_class": "measurement", "unit_of_measurement": "kWh", "icon": "mdi:solar-power"},
                 )
                 self.dashboard_item(
                     self.prefix + ".import_energy",

apps/predbat/predbat.py

@@ -36,7 +36,7 @@
 import requests
 import asyncio
 
-THIS_VERSION = "v8.25.3"
+THIS_VERSION = "v8.25.4"
 
 # fmt: off
 PREDBAT_FILES = ["predbat.py", "config.py", "prediction.py", "gecloud.py","utils.py", "inverter.py", "ha.py", "download.py", "unit_test.py", "web.py", "web_helper.py", "predheat.py", "futurerate.py", "octopus.py", "solcast.py","execute.py", "plan.py", "fetch.py", "output.py", "userinterface.py", "energydataservice.py", "alertfeed.py", "compare.py", "db_manager.py", "db_engine.py", "plugin_system.py", "ohme.py", "components.py", "fox.py"]
@@ -427,6 +427,9 @@ def reset(self):
         self.metric_inday_adjust_damping = 1.0
         self.metric_standing_charge = 0.0
         self.metric_self_sufficiency = 0.0
+        self.metric_dynamic_load_adjust = False
+        self.metric_pv_calibration_enable = True
+        self.dynamic_load_baseline = {}
         self.iboost_value_scaling = 1.0
         self.rate_import = {}
         self.rate_export = {}
@@ -526,6 +529,8 @@ def reset(self):
         self.load_minutes = {}
         self.load_minutes_now = 0
         self.load_minutes_age = 0
+        self.load_last_period = 0
+        self.load_last_status = "baseline"
         self.battery_capacity_nominal = False
         self.releases = {}
         self.balance_inverters_enable = False
@@ -681,6 +686,7 @@ def update_pred(self, scheduled=True):
         self.fetch_config_options()
         self.fetch_sensor_data()
         self.fetch_inverter_data()
+        self.dynamic_load()
 
         recompute = False
         if not scheduled or not self.plan_valid:

apps/predbat/prediction.py

@@ -10,7 +10,7 @@
 
 from datetime import timedelta
 from config import PREDICT_STEP, RUN_EVERY, TIME_FORMAT
-from utils import remove_intersecting_windows, get_charge_rate_curve, get_discharge_rate_curve, find_charge_rate, calc_percent_limit
+from utils import remove_intersecting_windows, get_charge_rate_curve, get_discharge_rate_curve, find_charge_rate, calc_percent_limit, in_iboost_slot, in_car_slot
 
 
 # Only assign globals once to avoid re-creating them with processes are forked
@@ -366,47 +366,6 @@ def find_charge_window_optimised(self, charge_windows, charge_limit, is_export=F
                     charge_window_optimised[minute] = window_n
         return charge_window_optimised
 
-    def in_iboost_slot(self, minute):
-        """
-        Is the given minute inside a car slot
-        """
-        load_amount = 0
-
-        if self.iboost_plan:
-            for slot in self.iboost_plan:
-                start_minutes = slot["start"]
-                end_minutes = slot["end"]
-                kwh = slot["kwh"]
-                slot_minutes = end_minutes - start_minutes
-                slot_hours = slot_minutes / 60.0
-
-                # Return the load in that slot
-                if minute >= start_minutes and minute < end_minutes:
-                    load_amount = abs(kwh / slot_hours)
-                    break
-        return load_amount
-
-    def in_car_slot(self, minute):
-        """
-        Is the given minute inside a car slot
-        """
-        load_amount = [0 for car_n in range(self.num_cars)]
-
-        for car_n in range(self.num_cars):
-            if self.car_charging_slots[car_n]:
-                for slot in self.car_charging_slots[car_n]:
-                    start_minutes = slot["start"]
-                    end_minutes = slot["end"]
-                    kwh = slot["kwh"]
-                    slot_minutes = end_minutes - start_minutes
-                    slot_hours = slot_minutes / 60.0
-
-                    # Return the load in that slot
-                    if minute >= start_minutes and minute < end_minutes:
-                        load_amount[car_n] = abs(kwh / slot_hours)
-                        break
-        return load_amount
-
     def run_prediction(self, charge_limit, charge_window, export_window, export_limits, pv10, end_record, save=None, step=PREDICT_STEP, cache=False):
         """
         Run a prediction scenario given a charge limit, return the results
@@ -679,7 +638,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
             car_freeze = False
 
             if car_enable:
-                car_load = self.in_car_slot(minute_absolute)
+                car_load = in_car_slot(minute_absolute, self.num_cars, self.car_charging_slots)
 
                 # Car charging?
                 for car_n in range(self.num_cars):
@@ -724,7 +683,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
                 # IBoost solar diverter on load, don't do on discharge
                 # IBoost based on plan for given rates
                 if self.iboost_plan and (self.iboost_on_export or (export_window_n < 0)):
-                    iboost_load = self.in_iboost_slot(minute_absolute) * step / 60.0
+                    iboost_load = in_iboost_slot(minute_absolute, self.iboost_plan) * step / 60.0
                     iboost_amount = min(iboost_load, self.iboost_max_power * step, max(self.iboost_max_energy - iboost_today_kwh, 0))
 
                 # IBoost based on Predbat charging