GECloud fixes for Gateway

Smoothing data fix for after the last sample

Author: springfall2008

apps/predbat/execute.py

@@ -725,31 +725,76 @@ def fetch_inverter_data(self, create=True):
         self.reserve_current = dp3(self.reserve_current)
         self.reserve_current_percent = calc_percent_limit(self.reserve_current, self.soc_max)
 
-        self.log(
-            "Found {} inverters totals: min reserve {} current reserve {} soc_max {} soc {} charge rate {} kW discharge rate {} kW battery_rate_min {} w ac limit {} export limit {} kW loss charge {} % loss discharge {} % inverter loss {} %".format(
-                len(self.inverters),
-                self.reserve,
-                self.reserve_current,
-                self.soc_max,
-                self.soc_kw,
-                self.charge_rate_now * MINUTE_WATT,
-                self.discharge_rate_now * MINUTE_WATT,
-                self.battery_rate_min * MINUTE_WATT,
-                dp3(self.inverter_limit * MINUTE_WATT),
-                dp3(self.export_limit * MINUTE_WATT),
-                100 - int(self.battery_loss * 100),
-                100 - int(self.battery_loss_discharge * 100),
-                100 - int(self.inverter_loss * 100),
+        if self.debug_enable:
+            self.log(
+                "Found {} inverters totals: min reserve {} current reserve {} soc_max {} soc {} charge rate {} kW discharge rate {} kW battery_rate_min {} w ac limit {} export limit {} kW loss charge {} % loss discharge {} % inverter loss {} %".format(
+                    len(self.inverters),
+                    self.reserve,
+                    self.reserve_current,
+                    self.soc_max,
+                    self.soc_kw,
+                    self.charge_rate_now * MINUTE_WATT,
+                    self.discharge_rate_now * MINUTE_WATT,
+                    self.battery_rate_min * MINUTE_WATT,
+                    dp3(self.inverter_limit * MINUTE_WATT),
+                    dp3(self.export_limit * MINUTE_WATT),
+                    100 - int(self.battery_loss * 100),
+                    100 - int(self.battery_loss_discharge * 100),
+                    100 - int(self.inverter_loss * 100),
+                )
             )
-        )
 
         # Work out current charge limits and publish charge limit base
         self.charge_limit = [self.current_charge_limit * self.soc_max / 100.0 for i in range(len(self.charge_window))]
         self.charge_limit_percent = calc_percent_limit(self.charge_limit, self.soc_max)
         self.publish_charge_limit(self.charge_limit, self.charge_window, self.charge_limit_percent, best=False)
+        self.publish_inverter_data()
+
+    def publish_inverter_data(self):
+        """
+        Publish inverter data to dashboard
+        """
+        self.dashboard_item(
+            self.prefix + ".pv_power",
+            state=dp3(self.pv_power / 1000.0),
+            attributes={
+                "friendly_name": "Predicted PV Power",
+                "state_class": "measurement",
+                "unit_of_measurement": "kW",
+                "icon": "mdi:battery",
+            },
+        )
+        self.dashboard_item(
+            self.prefix + ".grid_power",
+            state=dp3(self.grid_power / 1000.0),
+            attributes={
+                "friendly_name": "Predicted Grid Power",
+                "state_class": "measurement",
+                "unit_of_measurement": "kW",
+                "icon": "mdi:battery",
+            },
+        )
+        self.dashboard_item(
+            self.prefix + ".load_power",
+            state=dp3(self.load_power / 1000.0),
+            attributes={
+                "friendly_name": "Predicted Load Power",
+                "state_class": "measurement",
+                "unit_of_measurement": "kW",
+                "icon": "mdi:battery",
+            },
+        )        
+        self.dashboard_item(
+            self.prefix + ".battery_power",
+            state=dp3(self.battery_power / 1000.0),
+            attributes={
+                "friendly_name": "Predicted Battery Power",
+                "state_class": "measurement",
+                "unit_of_measurement": "kW",
+                "icon": "mdi:battery",
+            },
+        )
 
-        self.log("Base charge window {}".format(self.window_as_text(self.charge_window, self.charge_limit_percent)))
-        self.log("Base export window {}".format(self.window_as_text(self.export_window, self.export_limits)))
 
     def balance_inverters(self):
         """

apps/predbat/fetch.py

@@ -532,6 +532,7 @@ def minute_data(
         prev_last_updated_time=None,
         last_state=0,
         attributes=False,
+        max_increment=MAX_INCREMENT,
     ):
         """
         Turns data from HA into a hash of data indexed by minute with the data being the value
@@ -540,8 +541,8 @@ def minute_data(
         mdata = {}
         adata = {}
         newest_state = 0
+        prev_state = 0
         newest_age = 999999
-        max_increment = MAX_INCREMENT
 
         # Bounds on the data we store
         minute_min = -days * 24 * 60
@@ -680,6 +681,7 @@ def minute_data(
 
             if minutes < newest_age:
                 newest_age = minutes
+                prev_state = newest_state
                 newest_state = state
 
             # Power to Energy
@@ -749,6 +751,21 @@ def minute_data(
 
         # If we only have a start time then fill the gaps with the last values
         if not to_key:
+            # Fill from last sample until now with interolation if enabled
+            if clean_increment and backwards:
+                last_sample_minute = 0
+                for minute in range(60 * 24 * days):
+                    if minute in mdata:
+                        last_sample_minute = minute
+                        break
+                last_sample_value = mdata[last_sample_minute]
+                last_but_one_minute_sample = mdata[last_sample_minute + 1] if (last_sample_minute + 1) in mdata else last_sample_value
+                step = last_sample_value - last_but_one_minute_sample
+                if last_sample_minute < 10*60 and last_sample_minute > 0:
+                    for minute in range(last_sample_minute):
+                        if minute not in mdata:
+                            mdata[minute] = dp4(newest_state + step * (last_sample_minute - minute))
+
             # Fill from last sample until now
             for minute in range(60 * 24 * days):
                 if backwards:

apps/predbat/gecloud.py

@@ -153,8 +153,10 @@
 EVC_BLACKLIST_COMMANDS = ["installation-mode", "perform-factory-reset", "rename-id-tag", "delete-id-tags", "change-randomised-delay-duration"]
 
 TIMEOUT = 240
-RETRIES = 5
+RETRIES = 10
+RETRY_FACTOR = 1
 MAX_THREADS = 2
+MAX_START_TIME = 10*60
 
 attribute_table = {
     "time": {"friendly_name": "Time", "icon": "mdi:clock", "unit_of_measurement": "Time", "state_class": "timestamp"},
@@ -224,7 +226,7 @@ def wait_api_started(self):
         """
         self.log("GECloud: Waiting for API to start")
         count = 0
-        while not self.api_started and count < 240:
+        while not self.api_started and count < MAX_START_TIME:
             time.sleep(1)
             count += 1
         if not self.api_started:
@@ -372,6 +374,7 @@ async def publish_info(self, device, device_info):
             attributes = {}
             if key == "info":
                 info = device_info[key]
+                last_updated = device_info.get("last_updated", None)
                 cap = info.get("battery", {}).get("nominal_capacity", None)
                 volt = info.get("battery", {}).get("nominal_voltage", None)
                 dod = info.get("battery", {}).get("depth_of_discharge", None)
@@ -389,6 +392,7 @@ async def publish_info(self, device, device_info):
                 self.base.dashboard_item(entity_name + "_battery_size", capacity, attributes=attribute_table.get("battery_size", {}), app="gecloud")
                 self.base.dashboard_item(entity_name + "_max_charge_rate", max_charge_rate, attributes=attribute_table.get("max_charge_rate", {}), app="gecloud")
                 self.base.dashboard_item(entity_name + "_battery_dod", dod, attributes=attribute_table.get("_battery_dod", {}), app="gecloud")
+                self.base.dashboard_item(entity_name + "_last_updated", last_updated, attributes=attribute_table.get("time", {}), app="gecloud")
 
     async def publish_evc_data(self, serial, evc_data):
         """
@@ -642,6 +646,7 @@ async def async_automatic_config(self, devices):
             return
 
         batteries = devices["battery"]
+        batteries_real = devices["battery"]
         num_inverters = len(batteries)
 
         if devices["gateway"]:
@@ -670,21 +675,34 @@ async def async_automatic_config(self, devices):
         self.base.args["charge_limit"] = ["number.predbat_gecloud_" + device + "_ac_charge_upper_percent_limit" for device in batteries]
         self.base.args["discharge_start_time"] = ["select.predbat_gecloud_" + device + "_dc_discharge_1_start_time" for device in batteries]
         self.base.args["discharge_end_time"] = ["select.predbat_gecloud_" + device + "_dc_discharge_1_end_time" for device in batteries]
-        self.base.args["discharge_target_soc"] = ["select.predbat_gecloud_" + device + "_dc_discharge_1_lower_soc_percent_limit" for device in batteries]
+        self.base.args["discharge_target_soc"] = ["number.predbat_gecloud_" + device + "_dc_discharge_1_lower_soc_percent_limit" for device in batteries]
         self.base.args["scheduled_charge_enable"] = ["switch.predbat_gecloud_" + device + "_ac_charge_enable" for device in batteries]
         self.base.args["scheduled_discharge_enable"] = ["switch.predbat_gecloud_" + device + "_enable_dc_discharge" for device in batteries]
         self.base.args["pause_mode"] = ["select.predbat_gecloud_" + device + "_pause_battery" for device in batteries]
         self.base.args["pause_start_time"] = ["select.predbat_gecloud_" + device + "_pause_battery_start_time" for device in batteries]
         self.base.args["pause_end_time"] = ["select.predbat_gecloud_" + device + "_pause_battery_end_time" for device in batteries]
+
+        # Do we have a charge power percentage setting?
+        for device in batteries:
+            registers = self.settings.get(device, {})
+            for key in registers:
+                reg_name = registers[key].get("name", None)
+                value = registers[key].get("value", None)
+                if "inverter_charge_power_percentage" in reg_name.lower():
+                    self.log("GECloud: Found charge power percentage setting for {}, using this".format(device))
+                    self.base.args["charge_rate_percent"] = ["number.predbat_gecloud_" + device + "_inverter_charge_power_percentage" for device in batteries]
+                    self.base.args["discharge_rate_percent"] = ["number.predbat_gecloud_" + device + "_inverter_discharge_power_percentage" for device in batteries]
+                    break
+
         if "givtcp_rest" in self.base.args:
             del self.base.args["givtcp_rest"]
-
+    
         # Use the first battery serial for the ge_cloud_serial (for status)
         self.base.args["ge_cloud_serial"] = devices["battery"][0]
 
         # reconfigure for EMS
         if devices["ems"]:
-            self.log("GECloud: EMS detected")
+            self.log("GECloud: EMS detected, using this for control")
             ems = devices["ems"]
             self.base.args["inverter_type"] = ["GEE" for _ in range(num_inverters)]
             self.base.args["ge_cloud_serial"] = ems
@@ -747,9 +765,6 @@ async def start(self):
         for device in device_list:
             self.pending_writes[device] = []
 
-        if self.automatic:
-            await self.async_automatic_config(devices_dict)
-
         seconds = 0
         while not self.stop_cloud and not self.base.fatal_error:
             try:
@@ -774,8 +789,10 @@ async def start(self):
                             self.settings[device] = await self.async_get_inverter_settings(device, first=False, previous=self.settings.get(device, {}))
                             await self.publish_registers(device, self.settings[device])
 
-                    # Real time control disable?
+                    # One shot tasks
                     if seconds == 0:
+                        if self.automatic:
+                            await self.async_automatic_config(devices_dict)
                         for device in device_list:
                             await self.enable_real_time_control(device, self.settings[device])
 
@@ -813,7 +830,7 @@ async def async_send_evc_command(self, uuid, command, params):
                     data = None
             if data:
                 break
-            await asyncio.sleep(1 * (retry + 1))
+            await asyncio.sleep(RETRY_FACTOR * (retry + 1))
         if data is None:
             self.log("Error: GECloud: Failed to send EVC command {} params {}".format(command, params))
         return data
@@ -835,7 +852,7 @@ async def async_read_inverter_setting(self, serial, setting_id):
         if serial in self.pending_writes:
             for pending in self.pending_writes[serial]:
                 if pending["setting_id"] == setting_id:
-                    return {"value": pending["value"]}
+                    return {"value": pending["value"], "context": "predbat"}
 
         for retry in range(RETRIES):
             data = await self.async_get_inverter_data(GE_API_INVERTER_READ_SETTING, serial, setting_id, post=True)
@@ -847,12 +864,12 @@ async def async_read_inverter_setting(self, serial, setting_id):
             elif data and data_value in [-1, -2, -5, -6]:
                 data = None
                 # Inverter timeout, try to spread requests out
-                await asyncio.sleep(random.random() * 2)
+                await asyncio.sleep(random.random() * (3 + retry))
             if data:
                 break
-            await asyncio.sleep(1 * (retry + 1))
+            await asyncio.sleep(RETRY_FACTOR * (retry + 1))
         if data is None:
-            self.log("Warn: GECloud: Device {} Failed to read inverter setting id {}".format(serial, setting_id))
+            self.log("Warn: GECloud: Device {} Failed to read inverter setting id {} got {}".format(serial, setting_id, data))
         return data
 
     async def async_write_inverter_setting(self, serial, setting_id, value):
@@ -865,15 +882,15 @@ async def async_write_inverter_setting(self, serial, setting_id, value):
                 serial,
                 setting_id,
                 post=True,
-                datain={"value": str(value), "context": "homeassistant"},
+                datain={"value": str(value), "context": "predbat"},
             )
             if data and "success" in data:
                 if not data["success"]:
                     data = None
             if data:
                 self.pending_writes[serial].append({"setting_id": setting_id, "value": value})
                 break
-            await asyncio.sleep(1 * (retry + 1))
+            await asyncio.sleep(RETRY_FACTOR * (retry + 1))
         if data is None:
             self.log("Warn: GECloud: Failed to write setting id {} value {}".format(setting_id, value))
         return data
@@ -1169,6 +1186,7 @@ async def async_get_devices(self):
             self.log("GECloud: Found device {}".format(device))
             inverter = device.get("inverter", {})
             serial = inverter.get("serial", None)
+            last_updated = inverter.get("last_updated", None)
             model = inverter.get("info", {}).get("model", "").lower()
             batteries = inverter.get("connections", {}).get("batteries", [])
             if serial:
@@ -1202,7 +1220,7 @@ async def async_get_inverter_data_retry(self, endpoint, serial="", setting_id=""
             data = await self.async_get_inverter_data(endpoint, serial, setting_id, post, datain, uuid, meter_ids, start_time=start_time, end_time=end_time, command=command, measurands=measurands)
             if data is not None:
                 break
-            await asyncio.sleep(1 * (retry + 1))
+            await asyncio.sleep(RETRY_FACTOR * (retry + 1))
         if data is None:
             self.log("Warn: GECloud: Failed to get data from {}".format(endpoint))
         return data
@@ -1237,7 +1255,7 @@ async def async_get_inverter_data(self, endpoint, serial="", setting_id="", post
             self.log(f"Warn: GECloud: Exception during request to {url}: {e}")
             self.failures_total += 1
             return None
-
+                
         try:
             data = response.json()
         except requests.exceptions.JSONDecodeError:
@@ -1397,7 +1415,6 @@ def download_ge_data(self, now_utc):
         self.export_today_now = self.export_today.get(0, 0) - self.export_today.get(self.minutes_now, 0)
         self.pv_today_now = self.pv_today.get(0, 0) - self.pv_today.get(self.minutes_now, 0)
 
-        self.log("GE GECloud load_last_period is {} kW".format(dp2(self.load_last_period)))
         # More up to date sensors for current values if set
         if "load_today" in self.args:
             load_minutes, load_minutes_age = self.minute_data_load(self.now_utc, "load_today", self.max_days_previous, required_unit="kWh", load_scaling=self.load_scaling)

apps/predbat/inverter.py

@@ -389,7 +389,7 @@ def __init__(self, base, id=0, quiet=False, rest_postCommand=None, rest_getData=
             tdiff = dp2(tdiff.seconds / 60 + tdiff.days * 60 * 24)
             if not quiet:
                 self.base.log("Invertor time {}, Predbat computer time {}, difference {} minutes".format(self.inverter_time, now_utc, tdiff))
-            if abs(tdiff) >= 10:
+            if abs(tdiff) >= 30:
                 self.base.log(
                     "Warn: Invertor time is {}, Predbat computer time {}, this is {} minutes skewed, Predbat may not function correctly, please fix this by updating your inverter time, checking HA is synchronising with your inverter, or fixing Predbat computer time zone".format(
                         self.inverter_time, now_utc, tdiff
@@ -1457,8 +1457,7 @@ def write_and_poll_switch(self, name, entity_id, new_value):
                 self.base.call_service_wrapper(service, entity_id=entity_id)
 
             self.sleep(self.inv_write_and_poll_sleep)
-            current_state = self.base.get_state_wrapper(entity_id=entity_id, refresh=True)
-            self.log("Switch {} is now {}".format(entity_id, current_state))
+            current_state = self.base.get_state_wrapper(entity_id=entity_id, refresh=domain != "sensor")
             if isinstance(current_state, str):
                 current_state = current_state.lower() in ["on", "enable", "true"]
 
@@ -1503,7 +1502,7 @@ def write_and_poll_value(self, name, entity_id, new_value, fuzzy=0, ignore_fail=
                 return True
 
             self.sleep(self.inv_write_and_poll_sleep)
-            current_state = self.base.get_state_wrapper(entity_id, refresh=True, required_unit=required_unit)
+            current_state = self.base.get_state_wrapper(entity_id, refresh=domain != "sensor", required_unit=required_unit)
             if isinstance(new_value, str):
                 matched = current_state == new_value
             else: