diff --git a/.cspell/custom-dictionary-workspace.txt b/.cspell/custom-dictionary-workspace.txt index 74862d0cc..c5b4daf79 100644 --- a/.cspell/custom-dictionary-workspace.txt +++ b/.cspell/custom-dictionary-workspace.txt @@ -140,6 +140,7 @@ deadbanding deadbeefdeadbeef dedup dedupe +deepcopied dend denorm derate @@ -282,6 +283,7 @@ hainterface halfhourly hanchu hanchuess +handback hanres HAOS harvi @@ -441,6 +443,10 @@ nocharge NODETAIL nodischarge nohup +nonboolean +nonfeasible +nonfinal +nonfinite noopener nord nordpool @@ -561,6 +567,7 @@ replayer resetmidnight resultid resultmid +retrievalstatus retryable reweight reweighted @@ -710,6 +717,7 @@ unflushed unindexed unnormalised unparseable +unpassed unpickled unpickles unpushed @@ -735,6 +743,7 @@ vvwwxxabcdefyyzzuu waf waitpid Wallbox +wallclock weakref weblink welink diff --git a/README.md b/README.md index bf30be104..82719b61e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # Predbat +> Experimental [joint EV/home-battery planning](docs/joint-site-planning.md) +> is opt-in and shadow-only. It never replaces the executable native plan. + ![image](https://github.com/springfall2008/batpred/actions/workflows/code-quality.yml/badge.svg) ![image](https://github.com/springfall2008/batpred/actions/workflows/publish-docs.yml/badge.svg) ![image](https://github.com/springfall2008/batpred/actions/workflows/pages/pages-build-deployment/badge.svg) diff --git a/apps/predbat/config.py b/apps/predbat/config.py index 7c7d39d4e..dcbaab94c 100644 --- a/apps/predbat/config.py +++ b/apps/predbat/config.py @@ -2604,6 +2604,7 @@ # Apps.yaml validation schema APPS_SCHEMA = { + "joint_site": {"type": "dict"}, "currency_symbols": {"type": "string|string_list"}, "db_enable": {"type": "boolean"}, "db_days": {"type": "integer"}, diff --git a/apps/predbat/execute.py b/apps/predbat/execute.py index 45342d0ce..d32612ddf 100644 --- a/apps/predbat/execute.py +++ b/apps/predbat/execute.py @@ -295,6 +295,14 @@ def apply_inverter_rates(self, inverter, intent, notify_charge=True, notify_disc inverter.adjust_discharge_rate(int(discharge_rate), notify=notify_discharge) def execute_plan(self): + # Joint apply owns this cycle only after an explicit opt-in. It dispatches + # exact current intents through the same Inverter entry point, rather than + # allowing legacy window merging to erase hold/charge transitions. + from joint_site_apply import execute_joint_site + + joint_status = execute_joint_site(self) + if joint_status is not None: + return joint_status # Per-inverter detail segments, assembled into the status text after the headline status is # resolved - see build_status_extra() for why they can't be concatenated inline. status_extra_parts = [] diff --git a/apps/predbat/fetch.py b/apps/predbat/fetch.py index 3220b69cc..6c53be4d4 100644 --- a/apps/predbat/fetch.py +++ b/apps/predbat/fetch.py @@ -1750,6 +1750,10 @@ def fetch_sensor_data_car_planning(self): self.log("Car {} charging is exclusive, will not plan other cars".format(car_n)) break + from joint_site_arrival import prepare_arrival_forecast + + prepare_arrival_forecast(self) + def car_planned_slot_entity(self, planned_entity_id_list, car_n, slot_entity_id): """ The sensor a car's planned dispatches are read from, where that is not its slot sensor. @@ -1773,6 +1777,11 @@ def fetch_sensor_data_cars(self, save=True): live plan; diagnostics that record state across cycles must be skipped in that case. """ + from joint_site_recurring import recurring_enabled + + joint_soc = recurring_enabled(self.args.get("joint_site")) + # The standing joint policy reads its own SOC after availability checks. + # Native zero entries are inactive model placeholders, not observed SOC. # Work out current car SoC and limit self.car_charging_loss = 1 - float(self.get_arg("car_charging_loss")) @@ -1784,7 +1793,7 @@ def fetch_sensor_data_cars(self, save=True): if car_n < len(self.car_charging_manual_soc) and self.car_charging_manual_soc[car_n]: car_postfix = "" if car_n == 0 else "_" + str(car_n) self.car_charging_soc[car_n] = self.get_arg("car_charging_manual_soc_kwh" + car_postfix, 0.0) - else: + elif not (joint_soc and car_n == 0): self.car_charging_soc[car_n] = (self.get_arg("car_charging_soc", 0.0, index=car_n) * self.car_charging_battery_size[car_n]) / 100.0 # Get octopus intelligent slot configuration - could be single value or list for multiple cars @@ -1887,7 +1896,7 @@ def fetch_sensor_data_cars(self, save=True): if size: self.car_charging_battery_size[car_n] = size # Recalculate car SoC now that battery size has been updated from Octopus data - if car_n < len(self.car_charging_manual_soc) and not self.car_charging_manual_soc[car_n]: + if car_n < len(self.car_charging_manual_soc) and not self.car_charging_manual_soc[car_n] and not (joint_soc and car_n == 0): self.car_charging_soc[car_n] = (self.get_arg("car_charging_soc", 0.0, index=car_n) * self.car_charging_battery_size[car_n]) / 100.0 if rate: # Octopus's figure only ever raises the configured rate, never lowers it. Where Octopus's catalogue @@ -1993,7 +2002,8 @@ def fetch_sensor_data_cars(self, save=True): # Log final car SoC (initialised before the IOG loop, updated per-car after Octopus battery_size is read) if self.num_cars: car_charging_limit_percent = [dp1(limit / size * 100) if size else 0 for limit, size in zip(self.car_charging_limit, self.car_charging_battery_size)] - self.log("Cars: SoC: {}kWh, Charge limit {}% ({}kWh), plan time {}, battery size {}kWh".format(self.car_charging_soc, car_charging_limit_percent, self.car_charging_limit, self.car_charging_plan_time, self.car_charging_battery_size)) + displayed_soc = ["joint-managed" if joint_soc and index == 0 else value for index, value in enumerate(self.car_charging_soc)] + self.log("Cars: SoC: {}kWh, Charge limit {}% ({}kWh), plan time {}, battery size {}kWh".format(displayed_soc, car_charging_limit_percent, self.car_charging_limit, self.car_charging_plan_time, self.car_charging_battery_size)) def fetch_pv_forecast(self): """ diff --git a/apps/predbat/ha.py b/apps/predbat/ha.py index 141620ac7..70083df2d 100644 --- a/apps/predbat/ha.py +++ b/apps/predbat/ha.py @@ -612,7 +612,7 @@ async def socketLoop(self): if new_state: entity_id = new_state.get("entity_id", None) if entity_id: - self.update_state_item(new_state, entity_id) + self.update_state_item(new_state, entity_id, old_state=old_state) # Only trigger on value change or you get too many updates if not old_state or (new_state.get("state", None) != old_state.get("state", None)): await self.base.trigger_watch_list(entity_id, event_data.get("attribute", None), event_data.get("old_state", None), new_state) @@ -844,7 +844,7 @@ def update_state(self, entity_id): if item: self.update_state_item(item, entity_id) - def update_state_item(self, item, entity_id, nodb=False): + def update_state_item(self, item, entity_id, nodb=False, *, old_state=None, prior_snapshot=None): """ Update state table for item """ @@ -853,7 +853,12 @@ def update_state_item(self, item, entity_id, nodb=False): last_changed = item.get("last_changed", item.get("last_updated", None)) if "state" in item: state = item["state"] - self.state_data[entity_id] = {"state": state, "attributes": attributes, "last_changed": last_changed} + prior = prior_snapshot if prior_snapshot is not None else self.state_data.get(entity_id) + last_updated = item.get("last_updated", last_changed) + transition, history = HAInterface._value_transition(prior, item, old_state) + # Raw observation times are never rewritten. The additional metadata + # distinguishes proven same-state continuity from a lost event chain. + self.state_data[entity_id] = {"state": state, "attributes": attributes, "last_changed": last_changed, "last_updated": last_updated, "value_last_changed": transition, "value_history": history} if not nodb and self.db_enable and ((self.db_mirror_ha and (entity_id in self.db_mirror_list)) or self.db_primary): # Instead of appending to a local mirror_updates list, call the database manager to schedule the update if last_changed: @@ -871,6 +876,22 @@ def update_state_item(self, item, entity_id, nodb=False): self.db_manager.set_state_db(entity_id, state, attributes, timestamp=last_changed) + @staticmethod + def _value_transition(prior, item, old_state): + """Keep bounded real-value transitions only across an exactly adjacent HA event chain.""" + state, stamp = item.get("state"), item.get("last_changed", item.get("last_updated")) + entry = {"state": state, "at": stamp} + if not isinstance(prior, dict): + return stamp, [entry] + chained = isinstance(old_state, dict) and all(isinstance(prior.get(key), str) and old_state.get(key) == prior[key] for key in ("state", "last_changed", "last_updated")) + identical = old_state is None and all(item.get(key) == prior.get(key) for key in ("state", "last_changed", "last_updated")) + if chained or identical: + history = list(prior.get("value_history", [{"state": prior.get("state"), "at": prior.get("last_changed")}])) + if state != prior.get("state"): + return stamp, (history + [entry])[-16:] + return prior.get("value_last_changed") or prior.get("last_changed"), history + return None, [entry] + def update_states_db(self): """ Update the state data from the SQL Lite database @@ -899,11 +920,12 @@ def update_states(self): res = self.api_call("/api/states") if res: + prior = self.state_data self.state_data = {} for item in res: entity_id = item.get("entity_id", None) if entity_id: - self.update_state_item(item, entity_id) + self.update_state_item(item, entity_id, prior_snapshot=prior.get(entity_id.lower())) else: self.log("Warn: Failed to update state data from HA, item has no entity_id {}".format(item)) else: diff --git a/apps/predbat/joint_site.py b/apps/predbat/joint_site.py new file mode 100644 index 000000000..9d8e9c963 --- /dev/null +++ b/apps/predbat/joint_site.py @@ -0,0 +1,604 @@ +"""Pure bounded joint EV/battery scheduling; no HA, file, network or actuator access. + +All powers are AC kW at the site bus, except stored energies, which are cell-side +kWh. Demand mode is battery-first, hold permits PV charging only, and charge uses +the full configured charging rate. Battery saturation, reserve exhaustion, EV +completion and departure split a step into constant-power segments. Caps are +checked on every segment, never on an average that hides a short power peak. + +This is a single-EV AC-equivalent search model, not a hardware controller or a +proof of global optimality. Expected availability never establishes connection. +""" + +from dataclasses import dataclass, replace +import math +from typing import Optional, Sequence, Tuple + +STATUS_FEASIBLE = "feasible" +STATUS_BOUNDED_NO_PLAN = "bounded_no_plan" +BATTERY_DEMAND, BATTERY_HOLD, BATTERY_CHARGE = "demand", "hold", "charge" +BATTERY_MODES = (BATTERY_DEMAND, BATTERY_HOLD, BATTERY_CHARGE) +BATTERY_LIMITED_DEMAND = "limited_demand" +EV_OFF, EV_ON = "off", "on" +EPSILON = 1e-9 + + +@dataclass(frozen=True) +class Slot: + """Constant-power/rate interval; house and PV are AC kWh over duration_min.""" + + index: int + start_min: int + duration_min: int + ac_load_kwh: float + pv_kwh: float + import_price: float + export_price: float + battery_min_kwh: float = 0 + + +@dataclass(frozen=True) +class EVConfig: + """Fixed EV rate, cell-side energy request and expected availability indices. + + Pause/restart percentages refer to the DOMESTIC battery, not the EV. An + OFF-to-ON transition requires restart SoC and the minimum off interval; + continuing ON requires strictly more than pause SoC. Initial relay state is + explicit modelling input, never inferred from an expected arrival. The default + models a car already off for ten minutes, not permission to energise it. + """ + + battery_kwh: float + start_soc_kwh: float + stop_soc_kwh: float + rate_kw: float = 6.77 + efficiency: float = 0.92 + deadline_min: int = 0 + availability: Optional[Tuple[int, ...]] = None + pause_battery_percent: float = 15 + restart_battery_percent: float = 25 + min_off_minutes: float = 10 + initially_on: bool = False + initial_off_minutes: float = 10 + + +@dataclass(frozen=True) +class BatteryConfig: + """Cell-side energy bounds, AC rate limits and AC/cell efficiencies. + + Cycle cost is a configured wear surrogate per AC kWh discharged. Terminal + value is per stored kWh; the objective subtracts that credit exactly once. + """ + + capacity_kwh: float + initial_kwh: float + reserve_kwh: float + max_charge_kw: float + max_discharge_kw: float + charge_efficiency: float + discharge_efficiency: float + end_min_kwh: float + cycle_cost_per_kwh: float = 0 + terminal_value_per_kwh: float = 0 + + +@dataclass(frozen=True) +class SiteConfig: + """Whole-site AC grid caps, separately from an optional house-plus-EV load-port cap.""" + + import_cap_kw: float + export_cap_kw: Optional[float] = None + load_port_cap_kw: Optional[float] = None + charge_import_cap_kw: Optional[float] = None + + +@dataclass(frozen=True) +class PlannerConfig: + """Bounded search controls; beam=None retains exact states for at most 12 slots. + + Bucketing merges search labels, never stored energy. Such merging and beam + pruning can miss a feasible/cheaper plan, so a miss is not infeasibility proof. + + `allowed_battery_modes` declares the subset of battery dispatch modes the + planner may emit. It must be a tuple or list of string tags drawn from + the supported battery-mode vocabulary (`demand`, `hold`, `charge`). The + default equals the full base vocabulary; a tighter subset is honoured by + `_validate` and the search loop. When `allow_limited_discharge` is true, + `limited_demand` is added on top of whatever subset is declared here. + The planner never returns a mode that is not in this subset. + """ + + time_step_minutes: int = 5 + beam_width: Optional[int] = 128 + soc_bucket_kwh: float = 0.05 + allow_limited_discharge: bool = False + allowed_battery_modes: Tuple[str, ...] = BATTERY_MODES + economic_diversity: bool = False + initial_battery_mode: Optional[str] = None + require_ev_start: bool = False + solar_ev_only: bool = False + + +@dataclass(frozen=True) +class FlowSegment: + """One constant-power portion of a slot, including instantaneous constraint evidence.""" + + duration_min: float + load_kw: float + ev_kw: float + pv_kw: float + battery_charge_kw: float + battery_discharge_kw: float + grid_import_kw: float + grid_export_kw: float + curtailed_kw: float + soc_start_kwh: float + soc_end_kwh: float + + +@dataclass(frozen=True) +class SlotAction: + """Integrated slot flows; load_kwh excludes EV, which is reported separately.""" + + slot_index: int + battery_mode: str + ev_action: str + load_kwh: float + pv_kwh: float + pv_to_load_kwh: float + pv_to_battery_kwh: float + pv_curtailed_kwh: float + pv_exported_kwh: float + grid_import_kwh: float + grid_export_kwh: float + battery_discharge_kwh: float + battery_charge_kwh: float + battery_soc_kwh_start: float + battery_soc_kwh_end: float + ev_soc_kwh_start: float + ev_soc_kwh_end: float + ev_delivered_kwh: float + import_price: float + export_price: float + slot_cost: float + ev_on_minutes: float + segments: Tuple[FlowSegment, ...] + discharge_limit_kw: Optional[float] = None + + +@dataclass(frozen=True) +class Diagnostics: + """Completion, physical accounting and deterministic search-work evidence.""" + + unmet_load_kwh: float = 0 + pv_curtailed_total_kwh: float = 0 + ev_delivered_total_kwh: float = 0 + final_battery_soc_kwh: float = 0 + final_ev_soc_kwh: float = 0 + ev_stop_target_met: bool = False + battery_end_min_met: bool = False + ev_on_slot_count: int = 0 + mode_counts: Tuple[Tuple[str, int], ...] = () + notes: Tuple[str, ...] = () + transitions_evaluated: int = 0 + max_frontier_size: int = 0 + + +@dataclass(frozen=True) +class PlanResult: + """A complete feasible candidate or an explicit search miss with no proposed actions.""" + + status: str + total_cost: Optional[float] + actions: Tuple[SlotAction, ...] + battery_soc_trace: Tuple[float, ...] + ev_soc_trace: Tuple[float, ...] + import_kwh_trace: Tuple[float, ...] + export_kwh_trace: Tuple[float, ...] + curtail_kwh_trace: Tuple[float, ...] + diagnostics: Diagnostics + deadline_slot_index: int + + +class PlannerInputError(ValueError): + """Malformed inputs, not a verdict that a correctly formed request is infeasible.""" + + +def _require_finite(name, value, low=None, high=None): + """Validate without coercing strings/booleans or echoing private input values.""" + if type(value) not in (int, float) or not math.isfinite(value) or (low is not None and value < low) or (high is not None and value > high): + raise PlannerInputError("Invalid numeric field: " + name) + + +def _validate(slots, ev, battery, site, planner): + """Check units' domains, chronological coverage, state and bounded-search configuration.""" + if not isinstance(ev, EVConfig) or not isinstance(battery, BatteryConfig) or not isinstance(site, SiteConfig) or not isinstance(planner, PlannerConfig): + raise PlannerInputError("Expected typed planner configuration") + if type(planner.time_step_minutes) is not int or planner.time_step_minutes not in (5, 15, 30): + raise PlannerInputError("Unsupported planning step") + if planner.beam_width is not None and (type(planner.beam_width) is not int or not 1 <= planner.beam_width <= 4096): + raise PlannerInputError("beam_width must be an integer in 1..4096 or None") + if type(planner.allow_limited_discharge) is not bool: + raise PlannerInputError("allow_limited_discharge must be boolean") + if type(planner.require_ev_start) is not bool: + raise PlannerInputError("require_ev_start must be boolean") + if type(planner.solar_ev_only) is not bool: + raise PlannerInputError("solar_ev_only must be boolean") + allowed = planner.allowed_battery_modes + if not isinstance(allowed, (tuple, list)) or isinstance(allowed, (str, bytes)): + raise PlannerInputError("allowed_battery_modes must be a tuple or list") + if not allowed: + raise PlannerInputError("allowed_battery_modes must not be empty") + seen = set() + for mode in allowed: + if type(mode) is not str: + raise PlannerInputError("allowed_battery_modes entries must be strings") + if mode not in BATTERY_MODES: + raise PlannerInputError("allowed_battery_modes contains unsupported mode") + if mode in seen: + raise PlannerInputError("allowed_battery_modes contains duplicate mode") + seen.add(mode) + if planner.initial_battery_mode is not None and (type(planner.initial_battery_mode) is not str or planner.initial_battery_mode not in allowed): + raise PlannerInputError("initial_battery_mode must be an allowed mode or None") + _require_finite("soc_bucket_kwh", planner.soc_bucket_kwh, 1e-6) + if not slots or len(slots) > 576 or (planner.beam_width is None and len(slots) > 12): + raise PlannerInputError("Horizon must have 1..576 slots (at most 12 for exhaustive search)") + end = None + for index, slot in enumerate(slots): + if not isinstance(slot, Slot) or type(slot.index) is not int or slot.index != index: + raise PlannerInputError("Slot indices must be consecutive") + if type(slot.start_min) is not int or slot.start_min < 0 or type(slot.duration_min) is not int or slot.duration_min <= 0 or slot.duration_min % planner.time_step_minutes: + raise PlannerInputError("Invalid slot time") + if end is not None and slot.start_min != end: + raise PlannerInputError("Slots must be chronological and contiguous") + end = slot.start_min + slot.duration_min + for field in ("ac_load_kwh", "pv_kwh"): + _require_finite(field, getattr(slot, field), 0) + for field in ("import_price", "export_price"): + _require_finite(field, getattr(slot, field)) + _require_finite("slot_battery_min", slot.battery_min_kwh, 0, battery.capacity_kwh) + _require_finite("capacity_kwh", battery.capacity_kwh, 1e-6) + for field in ("initial_kwh", "reserve_kwh", "end_min_kwh"): + _require_finite(field, getattr(battery, field), 0, battery.capacity_kwh) + if battery.initial_kwh < battery.reserve_kwh or battery.end_min_kwh < battery.reserve_kwh: + raise PlannerInputError("Battery energy and terminal floor must respect domestic reserve") + for field in ("max_charge_kw", "max_discharge_kw", "cycle_cost_per_kwh", "terminal_value_per_kwh"): + _require_finite(field, getattr(battery, field), 0) + for field in ("charge_efficiency", "discharge_efficiency"): + _require_finite(field, getattr(battery, field), 1e-6, 1) + _require_finite("ev_capacity", ev.battery_kwh, 1e-6) + for field in ("start_soc_kwh", "stop_soc_kwh"): + _require_finite(field, getattr(ev, field), 0, ev.battery_kwh) + _require_finite("ev_rate", ev.rate_kw, 1e-6) + _require_finite("ev_efficiency", ev.efficiency, 1e-6, 1) + _require_finite("pause_battery_percent", ev.pause_battery_percent, 0, 100) + _require_finite("restart_battery_percent", ev.restart_battery_percent, ev.pause_battery_percent, 100) + for field in ("min_off_minutes", "initial_off_minutes"): + _require_finite(field, getattr(ev, field), 0) + if type(ev.initially_on) is not bool or type(ev.deadline_min) is not int or ev.deadline_min < 0: + raise PlannerInputError("Invalid EV initial state or deadline") + if ev.availability is not None and (not isinstance(ev.availability, tuple) or any(type(i) is not int or not 0 <= i < len(slots) for i in ev.availability)): + raise PlannerInputError("Availability must contain original slot indices") + _require_finite("import_cap_kw", site.import_cap_kw, 0) + if site.charge_import_cap_kw is not None: + _require_finite("charge_import_cap_kw", site.charge_import_cap_kw, 1e-6, site.import_cap_kw) + for field in ("export_cap_kw", "load_port_cap_kw"): + if getattr(site, field) is not None: + _require_finite(field, getattr(site, field), 0) + + +def _ev_is_available(ev, slot_index): + """Expected availability only, deliberately unrelated to actual connection permission.""" + return ev.availability is None or slot_index in ev.availability + + +def _deadline_slot_index(slots, deadline_min): + """Last original slot containing any time before departure, including a partial interval.""" + return max((s.index for s in slots if s.start_min < deadline_min), default=-1) + + +def _bucket(value, width): + """Bucket search labels only; physical state values are never rounded or overwritten.""" + return math.floor(value / width) + + +@dataclass(slots=True) +class _Node: + """One retained search label, including the EV's execution-relevant temporal state.""" + + battery_kwh: float + ev_kwh: float + on: bool + off_minutes: float + cost: float = 0 + switches: int = 0 + parent: Optional["_Node"] = None + action: Optional[SlotAction] = None + + +def _segment(duration, house_kw, ev_kw, pv_kw, soc, mode, battery, ev, site): + """Simulate constant demand, splitting at full/reserve boundaries; reject any power excess.""" + segments = [] + remaining = duration / 60 + while remaining > 1e-12: + demand = house_kw + ev_kw + if site.load_port_cap_kw is not None and demand > site.load_port_cap_kw + EPSILON: + return None + charge = discharge = 0.0 + if soc < battery.capacity_kwh - EPSILON: + charge = battery.max_charge_kw if mode == BATTERY_CHARGE else min(max(pv_kw - demand, 0), battery.max_charge_kw) + if mode == BATTERY_CHARGE and site.charge_import_cap_kw is not None: + # A grid-charge REQUEST need not mean physical battery charging: + # house/EV loads take priority. Use headroom to charge, or supply + # the grid deficit from the battery. The site cap below still + # rejects insufficient support, including reserve exhaustion. + charge = min(charge, max(site.charge_import_cap_kw - demand + pv_kw, 0)) + if soc > battery.reserve_kwh + EPSILON: + discharge = min(max(demand - pv_kw - site.charge_import_cap_kw, 0), battery.max_discharge_kw) + if mode == BATTERY_DEMAND and soc > battery.reserve_kwh + EPSILON: + discharge = min(max(demand - pv_kw, 0), battery.max_discharge_kw) + net = demand + charge - pv_kw - discharge + imported = max(net, 0) + if imported > site.import_cap_kw + EPSILON: + return None + surplus = max(-net, 0) + exported = min(surplus, site.export_cap_kw) if site.export_cap_kw is not None else surplus + derivative = charge * battery.charge_efficiency - discharge / battery.discharge_efficiency + hours = remaining + if derivative > 0: + hours = min(hours, (battery.capacity_kwh - soc) / derivative) + elif derivative < 0: + hours = min(hours, (soc - battery.reserve_kwh) / -derivative) + next_soc = soc + hours * derivative + if ev_kw and min(soc, next_soc) <= battery.capacity_kwh * ev.pause_battery_percent / 100 + EPSILON: + return None + segments.append(FlowSegment(hours * 60, house_kw, ev_kw, pv_kw, charge, discharge, imported, exported, surplus - exported, soc, next_soc)) + remaining -= hours + soc = next_soc + return tuple(segments), soc + + +def _resolve_action(slot, ev_on, mode, node, battery, ev, site): + """Compute a complete physical step, never a partially served household/EV load.""" + on_minutes = 0.0 + discharge_limit = None + if ev_on: + if not _ev_is_available(ev, slot.index) or slot.start_min >= ev.deadline_min or node.ev_kwh >= ev.stop_soc_kwh - EPSILON: + return None + if not node.on and (node.off_minutes + EPSILON < ev.min_off_minutes or node.battery_kwh + EPSILON < battery.capacity_kwh * ev.restart_battery_percent / 100): + return None + on_minutes = min(slot.duration_min, ev.deadline_min - slot.start_min, (ev.stop_soc_kwh - node.ev_kwh) / ev.efficiency / ev.rate_kw * 60) + flow_mode = mode + if mode == BATTERY_LIMITED_DEMAND: + # Still battery-first, but under a reproducible per-period power ceiling. + # The fixed ceiling remains in place after a partly completed EV step; + # it is not a fictitious continuously modulating grid-first controller. + net_kw = (slot.ac_load_kwh - slot.pv_kwh) * 60 / slot.duration_min + (ev.rate_kw if on_minutes else 0) + discharge_limit = min(battery.max_discharge_kw, max(0, math.ceil((net_kw - site.import_cap_kw) * 1000 + 2) / 1000)) + battery = replace(battery, max_discharge_kw=discharge_limit) + flow_mode = BATTERY_DEMAND + segments = [] + soc = node.battery_kwh + for duration, ev_kw in ((on_minutes, ev.rate_kw), (slot.duration_min - on_minutes, 0)): + if duration <= 1e-10: + continue + result = _segment(duration, slot.ac_load_kwh * 60 / slot.duration_min, ev_kw, slot.pv_kwh * 60 / slot.duration_min, soc, flow_mode, battery, ev, site) + if result is None: + return None + parts, soc = result + segments.extend(parts) + if any(min(part.soc_start_kwh, part.soc_end_kwh) < slot.battery_min_kwh - EPSILON for part in segments): + return None + imported = sum(s.grid_import_kw * s.duration_min / 60 for s in segments) + exported = sum(s.grid_export_kw * s.duration_min / 60 for s in segments) + discharged = sum(s.battery_discharge_kw * s.duration_min / 60 for s in segments) + charged = sum(s.battery_charge_kw * s.duration_min / 60 for s in segments) + delivered = ev.rate_kw * on_minutes / 60 + pv_load = sum(min(s.pv_kw, s.load_kw + s.ev_kw) * s.duration_min / 60 for s in segments) + pv_charge = sum(min(max(s.pv_kw - s.load_kw - s.ev_kw, 0), s.battery_charge_kw) * s.duration_min / 60 for s in segments) + curtailed = sum(s.curtailed_kw * s.duration_min / 60 for s in segments) + cost = imported * slot.import_price - exported * slot.export_price + discharged * battery.cycle_cost_per_kwh + return SlotAction( + slot.index, + mode, + EV_ON if on_minutes else EV_OFF, + slot.ac_load_kwh, + slot.pv_kwh, + pv_load, + pv_charge, + curtailed, + exported, + imported, + exported, + discharged, + charged, + node.battery_kwh, + soc, + node.ev_kwh, + node.ev_kwh + delivered * ev.efficiency, + delivered, + slot.import_price, + slot.export_price, + cost, + on_minutes, + tuple(segments), + discharge_limit, + ) + + +def _prune(nodes, width, bucket, terminal_value, preserve_battery=False, economic_diversity=False): + """Preserve EV progress and optionally stored-energy resilience within the same beam bound. + + The fixed beam budget often causes a small number of distinct EV-progress + buckets to share it with many near-duplicate cost-ranked candidates. With no + hold action, the cheapest label of a bucket may sit on a stored-energy path + that prematurely imports during the current band and forecloses a cheap + later pre-charge. Promoting only the highest-energy neighbour (the + previous behaviour) keeps one extreme but can still drop the intermediate + stored-energy trajectory that completes the cheaper plan. + + When ``preserve_battery`` is true, the cost leader stays at position 0 of + its bucket (so the layer-by-layer budget picks it first across buckets). + With opt-in ``economic_diversity``, the positions up to ``width`` are re-sorted by stored + energy descending, so each per-bucket layer picks a different SoC level + instead of `width`-many near-duplicates. Beyond the head, ranking order + resumes unchanged. The frontier size and the per-slot evaluation budget + stay within ``width``; this is a within-bucket ordering change, never an + addition of new candidates. This remains bounded search, not an optimality proof. + """ + groups = {} + for node in nodes: + groups.setdefault(_bucket(node.ev_kwh, bucket), []).append(node) + for group in groups.values(): + group.sort(key=lambda n: (n.cost - terminal_value * n.battery_kwh, n.switches)) + if preserve_battery and len(group) > 2: + # Cost leader stays at position 0; the next ``width - 1`` + # candidates are re-ordered by stored energy descending so layer + # picks span SoC tiers instead of cost-ranked duplicates. Past + # the head, ranking order resumes so all cheap-end behaviour is + # unchanged. + leader = group[0] + high = max(group, key=lambda n: n.battery_kwh) + if high is not leader: + group.remove(high) + group.insert(1, high) + if economic_diversity: + head_rest = list(group[1 : max(1, width)]) + tail = list(group[max(1, width) :]) + head_rest.sort(key=lambda n: -n.battery_kwh) + group.clear() + group.extend([leader] + head_rest + tail) + ordered = [groups[key] for key in sorted(groups, reverse=True)] + kept = [] + layer = 0 + while len(kept) < width: + added = False + for group in ordered: + if layer < len(group): + kept.append(group[layer]) + added = True + if len(kept) == width: + break + if not added: + break + layer += 1 + return kept + + +def _bounded_no_plan(deadline_idx, transitions, max_frontier, reason): + """A bounded search miss is neither a partial schedule nor proof of physical impossibility.""" + diagnostic = Diagnostics(notes=(reason, "no_global_optimality_or_infeasibility_claim"), transitions_evaluated=transitions, max_frontier_size=max_frontier) + return PlanResult(STATUS_BOUNDED_NO_PLAN, None, (), (), (), (), (), (), diagnostic, deadline_idx) + + +def _reachable_floor_envelope(slots, battery, site): + """Optimistic charging bound that preserves time-local floors during pruning. + + Assume the EV is off and any grid/PV headroom can charge the battery, even + where an actual discrete action could not use all that headroom. This only + rejects states unable to reach a future floor under the most favourable + charging assumption; it does not claim a feasible continuation exists. + """ + required_end = [0.0] * len(slots) + required = battery.end_min_kwh + for slot in reversed(slots): + required = max(required, slot.battery_min_kwh, battery.reserve_kwh) + required_end[slot.index] = required + headroom = max(0, site.import_cap_kw + (slot.pv_kwh - slot.ac_load_kwh) * 60 / slot.duration_min) + gain = min(battery.max_charge_kw, headroom) * slot.duration_min / 60 * battery.charge_efficiency + required = max(slot.battery_min_kwh, battery.reserve_kwh, required - gain) + return required_end + + +def plan(slots: Sequence[Slot], ev: EVConfig, battery: BatteryConfig, site: SiteConfig, planner: Optional[PlannerConfig] = None) -> PlanResult: + """Jointly search all eligible prices, EV decisions and executable-mode physics. + + Hard constraints reject transitions before monetary ranking. All returned + candidates meet useful demand and the terminal floor. The objective includes + losses, actual grid purchases/exports, cycle cost and terminal stored energy. + """ + planner = planner if planner is not None else PlannerConfig() + _validate(slots, ev, battery, site, planner) + base_modes = tuple(planner.allowed_battery_modes) + modes = base_modes + ((BATTERY_LIMITED_DEMAND,) if planner.allow_limited_discharge and BATTERY_LIMITED_DEMAND not in base_modes else ()) + required_end = _reachable_floor_envelope(slots, battery, site) + deadline_idx = _deadline_slot_index(slots, ev.deadline_min) + future = [0.0] * (len(slots) + 1) + for i in range(len(slots) - 1, -1, -1): + slot = slots[i] + available_minutes = min(slot.duration_min, max(ev.deadline_min - slot.start_min, 0)) if _ev_is_available(ev, slot.index) else 0 + future[i] = future[i + 1] + available_minutes / 60 * ev.rate_kw * ev.efficiency + if ev.start_soc_kwh + future[0] < ev.stop_soc_kwh - EPSILON: + return _bounded_no_plan(deadline_idx, 0, 0, "insufficient_available_charging_time") + frontier = [_Node(battery.initial_kwh, ev.start_soc_kwh, ev.initially_on, min(ev.initial_off_minutes, ev.min_off_minutes))] + max_frontier = 1 + transitions = 0 + pruned = False + for i, slot in enumerate(slots): + next_states = {} + for node in frontier: + for mode in (planner.initial_battery_mode,) if i == 0 and planner.initial_battery_mode is not None else modes: + for on in (True,) if i == 0 and planner.require_ev_start else (False, True): + if on and planner.solar_ev_only and (mode != BATTERY_DEMAND or slot.pv_kwh - slot.ac_load_kwh < ev.rate_kw * slot.duration_min / 60 - EPSILON): + continue + transitions += 1 + action = _resolve_action(slot, on, mode, node, battery, ev, site) + if action is None: + continue + if action.battery_soc_kwh_end < required_end[i] - EPSILON: + continue + if action.ev_soc_kwh_end + future[i + 1] < ev.stop_soc_kwh - EPSILON: + continue + if i == len(slots) - 1 and action.battery_soc_kwh_end < battery.end_min_kwh - EPSILON: + continue + on_end = action.ev_on_minutes >= slot.duration_min - EPSILON and action.ev_on_minutes > 0 + off_minutes = slot.duration_min - action.ev_on_minutes if action.ev_on_minutes else (0 if node.on else node.off_minutes) + slot.duration_min + off_minutes = 0 if on_end else min(off_minutes, ev.min_off_minutes) + new = _Node(action.battery_soc_kwh_end, action.ev_soc_kwh_end, on_end, off_minutes, node.cost + action.slot_cost, node.switches + int(on != node.on), node, action) + # Exact search does not bucket unequal physical states together. + energy_key = (new.battery_kwh, new.ev_kwh) if planner.beam_width is None else (_bucket(new.battery_kwh, planner.soc_bucket_kwh), _bucket(new.ev_kwh, planner.soc_bucket_kwh)) + key = energy_key + (on_end, off_minutes) + existing = next_states.get(key) + rank = (new.cost - battery.terminal_value_per_kwh * new.battery_kwh, new.switches) + if existing is None or rank < (existing.cost - battery.terminal_value_per_kwh * existing.battery_kwh, existing.switches): + next_states[key] = new + if not next_states: + return _bounded_no_plan(deadline_idx, transitions, max_frontier, "no_complete_transition_at_slot_{}".format(i)) + frontier = list(next_states.values()) + if planner.beam_width is not None and len(frontier) > planner.beam_width: + frontier = _prune(frontier, planner.beam_width, planner.soc_bucket_kwh, battery.terminal_value_per_kwh, preserve_battery=BATTERY_HOLD not in base_modes, economic_diversity=planner.economic_diversity) + pruned = True + max_frontier = max(max_frontier, len(frontier)) + best = min(frontier, key=lambda n: (n.cost - battery.terminal_value_per_kwh * n.battery_kwh, n.switches)) + actions = [] + node = best + while node.action is not None: + actions.append(node.action) + node = node.parent + actions = tuple(reversed(actions)) + notes = ("beam_pruned" if pruned else "beam_not_pruned", "bucketed_search" if planner.beam_width is not None else "exact_discrete_search", "no_global_optimality_claim") + diagnostic = Diagnostics( + pv_curtailed_total_kwh=sum(a.pv_curtailed_kwh for a in actions), + ev_delivered_total_kwh=sum(a.ev_delivered_kwh for a in actions), + final_battery_soc_kwh=best.battery_kwh, + final_ev_soc_kwh=best.ev_kwh, + ev_stop_target_met=True, + battery_end_min_met=True, + ev_on_slot_count=sum(a.ev_action == EV_ON for a in actions), + mode_counts=tuple((mode, sum(a.battery_mode == mode for a in actions)) for mode in modes), + notes=notes, + transitions_evaluated=transitions, + max_frontier_size=max_frontier, + ) + return PlanResult( + STATUS_FEASIBLE, + best.cost - battery.terminal_value_per_kwh * best.battery_kwh, + actions, + (battery.initial_kwh,) + tuple(a.battery_soc_kwh_end for a in actions), + (ev.start_soc_kwh,) + tuple(a.ev_soc_kwh_end for a in actions), + tuple(a.grid_import_kwh for a in actions), + tuple(a.grid_export_kwh for a in actions), + tuple(a.pv_curtailed_kwh for a in actions), + diagnostic, + deadline_idx, + ) diff --git a/apps/predbat/joint_site_apply.py b/apps/predbat/joint_site_apply.py new file mode 100644 index 000000000..95f7b3924 --- /dev/null +++ b/apps/predbat/joint_site_apply.py @@ -0,0 +1,670 @@ +"""Explicit joint execution through the native inverter and the sole external EV executor. + +Off and shadow are strict no-ops. HA observations are never hardware confirmation. +This module neither sends EV switch services nor changes standing inverter limits. +""" + +from copy import copy, deepcopy +from datetime import datetime, timedelta +import hashlib +import json +import math + +from const import PREDBAT_MAX_CARS, PREDICT_STEP, MINUTE_WATT + +RECEIPT_TTL_SECONDS = 300 +PLAN_REVALIDATION_MAX_AGE_SECONDS = 600 +OBSERVATION_MAX_AGE_SECONDS = 180 +EV_HANDOFF_SECONDS = 30 +SOLAR_RECEIPT_SECONDS = 60 +SOLAR_OBSERVATION_SECONDS = 60 + + +class JointBlocked(ValueError): + """Carry a public reason code, never raw configuration or transport details.""" + + +def _require(condition, reason): + """Fail closed on a violated execution contract.""" + if not condition: + raise JointBlocked(reason) + + +def _clock(): + """Use the same real observation clock as the native input adapter.""" + from joint_site_native import _observation_clock, _aware + + return _aware(_observation_clock(), "execution_clock") + + +def _stamp(value): + """Read a strictly offset-aware timestamp through the native parser.""" + from joint_site_native import _parse_entity_timestamp, _aware + + return _aware(value, "execution_timestamp") if isinstance(value, datetime) else _parse_entity_timestamp(value, "execution_timestamp") + + +def _finite(value, minimum=0, maximum=None): + """Accept finite native numeric values without coercing booleans or strings.""" + return type(value) in (int, float) and math.isfinite(value) and value >= minimum and (maximum is None or value <= maximum) + + +def _requested(base): + """Recognise only the explicit apply setting without touching runtime state.""" + settings = base.args.get("joint_site") + return isinstance(settings, dict) and settings.get("mode") == "apply" + + +def native_plan_hook_allowed(base): + """Do not broadcast independent legacy controls as an executed joint plan.""" + return not _requested(base) + + +def execution_fingerprint(base): + """Bind staging to actual configuration, not to the pending object's own copy.""" + names = ( + "num_cars", + "num_inverters", + "soc_max", + "reserve", + "import_limit", + "export_limit", + "inverter_limit", + "battery_rate_max_charge", + "battery_rate_max_charge_dc", + "battery_rate_max_discharge", + "battery_rate_max_scaling", + "battery_rate_max_scaling_discharge", + "inverter_loss", + "battery_loss", + "battery_loss_discharge", + "car_charging_rate", + "car_charging_limit", + "car_charging_battery_size", + "car_charging_loss", + "car_charging_manual_soc", + "car_charging_from_battery", + "car_energy_reported_load", + "set_charge_window", + "set_charge_freeze", + "set_discharge_during_charge", + "set_charge_freeze_only", + "set_charge_low_power", + "set_read_only", + "set_read_only_axle", + "iboost_enable", + "manual_charge_times", + "manual_demand_times", + "manual_export_times", + "manual_freeze_charge_times", + "manual_freeze_export_times", + "all_active_keep", + "all_active_keep_max", + "manual_soc_max_keep", + "io_adjusted", + ) + payload = {name: getattr(base, name, None) for name in names} + inverters = getattr(base, "inverters", None) + if isinstance(inverters, list) and len(inverters) == 1 and getattr(inverters[0], "inverter_type", None) == "DeyeSolarman" and getattr(inverters[0], "inv_support_charge_freeze", None) is False: + # The native config fetch restores the user's preference before the + # inverter fetch disables unsupported charge freeze again. Publication + # runs between those stages: fingerprint the SAME effective capability + # used by native Prediction, not that transient unsupported preference. + # _evidence/validate_template independently reject changed capabilities. + payload["set_charge_freeze"] = False + payload["settings"] = base.args.get("joint_site") + from joint_site_native import _ev_charging_priority + + payload["ev_charging_priority"] = _ev_charging_priority(base) + if payload["ev_charging_priority"] == "solar": + payload["solar_meter_mapping"] = {name: base.args.get(name) for name in ("pv_power", "load_power", "load_power_invert", "deye_entity_prefix")} + payload["solar_conversion"] = {name: getattr(base, name, None) for name in ("inverter_hybrid", "pv_ac_limit")} + from joint_site_recurring import recurring_enabled, resolve_recurring_session + + if recurring_enabled(payload["settings"]): + resolved = resolve_recurring_session(base) + payload["recurring_session"] = {"identity": resolved.get("identity"), "departure": resolved.get("session", {}).get("departure"), "reason": resolved.get("reason")} + payload["recurring_ready_time"] = base.car_charging_plan_time + payload["recurring_timezone"] = base.args.get("timezone") + payload["deye_local"] = base.args.get("deye_local") + if isinstance(payload["settings"], dict) and payload["settings"].get("control_model") == "deye_solarman": + payload["template_controls"] = {name: base.args.get(name) for name in ("inverter", "inverter_type", "deye_entity_prefix", "charge_start_service", "charge_stop_service", "discharge_stop_service", "charge_freeze_service")} + encoded = json.dumps(payload, sort_keys=True, allow_nan=False, default=str).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _cached(base, entity): + """Require an explicitly configured cached entity; never refresh hardware.""" + from joint_site_native import _read_entity_state + + _require(isinstance(entity, str) and bool(entity), "entity_mapping_required") + result = _read_entity_state(base, entity) + _require(result is not None, "entity_observation_unavailable") + return result + + +def command_fingerprint(commands): + """Bind exact staged commands to the paired replay; a valid but different target is not accepted.""" + return hashlib.sha256(json.dumps(commands, sort_keys=True, allow_nan=False).encode()).hexdigest() + + +def _control_fingerprint(inverter): + """Detect observed control drift without claiming read-back proves hardware state.""" + if getattr(inverter, "inverter_type", None) == "DeyeSolarman": + from joint_site_template import snapshot_template + + snapshot = snapshot_template(inverter) + else: + snapshot = inverter.deye_local.snapshot() + return hashlib.sha256(json.dumps(snapshot, sort_keys=True, allow_nan=False).encode()).hexdigest() + + +def _acknowledged_stop_state(settings, connection, now): + """Accept only the exact retained error explicitly attributed to the operator's manual stop. + + This is not an error-state allowlist. Any subsequent state/attribute publication, + changed departure or absent confirmation loses this exception; BMS, metering and + all other executor guards remain authoritative. + """ + acknowledgement = settings.get("connection_error_acknowledgement") + if acknowledgement is None: + return False + _require(settings.get("control_model") == "deye_solarman" and settings.get("confirmed_session") and settings.get("grid_charge_support"), "stop_acknowledgement_scope") + _require(isinstance(acknowledgement, dict) and set(acknowledgement) == {"state", "observed_at", "acknowledged_at", "departure"}, "stop_acknowledgement_invalid") + _require(acknowledgement["state"] == "4", "stop_acknowledgement_invalid") + observed = _stamp(acknowledgement["observed_at"]) + accepted = _stamp(acknowledgement["acknowledged_at"]) + departure = _stamp(acknowledgement["departure"]) + _require(_stamp(settings["session"]["available_from"]) <= observed <= accepted <= now, "stop_acknowledgement_invalid") + _require(departure == _stamp(settings["confirmed_session"]["departure"]) == _stamp(settings["session"]["departure"]), "stop_acknowledgement_invalid") + return now < departure and connection is not None and connection["state"] == "4" and connection["last_updated"] == observed + + +def _evidence(base, allow_read_only_transition=False): + """Revalidate actual capabilities, overrides and independent EV evidence before use.""" + from joint_site_native import _options, _collect_blockers + + settings = base.args["joint_site"] + from joint_site_session import household_reason + from joint_site_recurring import recurring_enabled + + _require(household_reason(base) is None, "ev_unavailable_for_joint_execution") + # Explicit experimental session authority is not a claim that the hardware + # commissioning attestation is true. All electrical/control gates below remain. + metered_session = settings.get("confirmed_session") is not None or recurring_enabled(settings) + authorised_session = settings.get("execution_authorised") is True and metered_session and settings.get("grid_charge_support") is not None and settings.get("control_model") == "deye_solarman" + _require(settings.get("apply_verified") is True or authorised_session, "apply_commissioning_required") + _require(settings.get("ev_executor_lease_verified") is True, "ev_executor_lease_required") + _require(not getattr(base, "octopus_intelligent_charging", False), "external_ev_scheduler_unsupported") + _require(not base.get_arg("octopus_intelligent_slot", None, indirect=False), "external_ev_scheduler_unsupported") + hooks = getattr(getattr(base, "plugin_system", None), "hooks", {}) + _require(not hooks.get("on_plan_executed"), "external_plan_consumer_unsupported") + if not metered_session: + stamps = [_stamp(_cached(base, settings.get(name))["state"]) for name in ("soc_observed_at_entity", "connection_observed_at_entity")] + now = _clock() + _require(all(0 <= (now - stamp).total_seconds() <= OBSERVATION_MAX_AGE_SECONDS for stamp in stamps), "physical_observation_stale_or_future") + options = _options(base) + _require(options is not None and options["mode"] == "apply" and options["control_model"] in ("deye_tou", "deye_solarman"), "real_deye_controls_required") + _require(not _collect_blockers(base, options), "native_inputs_or_overrides_blocked") + _require(not base.set_read_only and not getattr(base, "set_read_only_axle", False), "read_only") + transition = allow_read_only_transition and getattr(base, "inverter_needs_reset_force", "") == "set_read_only" and options["control_model"] == "deye_solarman" and options.get("grid_charge_support_w") and options.get("confirmed_observation") + _require(not getattr(base, "inverter_needs_reset", False) or transition, "native_state_not_ready") + _require(not any(getattr(base, name, False) for name in ("had_errors", "fatal_error")), "native_state_not_ready") + _require(not base.get_arg("template", False), "template_mode") + inverters = getattr(base, "inverters", None) + _require(isinstance(inverters, list) and len(inverters) == 1, "single_inverter_required") + inverter = inverters[0] + if options["control_model"] == "deye_solarman": + from joint_site_template import validate_template + + _require(getattr(inverter, "inverter_type", None) == "DeyeSolarman", "template_inverter_required") + try: + validate_template(inverter) + except (ValueError, TypeError, KeyError, AttributeError): + raise JointBlocked("template_contract_invalid") from None + _require(not getattr(inverter, "in_calibration", True), "inverter_capability_or_calibration") + else: + _require(getattr(inverter, "inverter_type", None) == "DEYE", "local_deye_required") + adapter = getattr(inverter, "deye_local", None) + _require(adapter is not None, "local_deye_adapter_missing") + _require(adapter.settings.get("control") is True and adapter.settings.get("hold_verified") is True, "deye_commissioning_required") + _require(getattr(inverter, "inv_support_charge_freeze", False) is True and not getattr(inverter, "in_calibration", True), "inverter_capability_or_calibration") + _require(not getattr(base, "manual_soc_max_keep", None), "manual_ceiling_unsupported") + _require(base.export_limit == 0 and base.set_discharge_during_charge, "deye_dispatch_model_mismatch") + _require(options["import_limit_w"] == getattr(base, "import_limit", options["import_limit_w"]), "import_limit_mismatch") + _require(options["ev_soc_source"] == "entity_cache" or options.get("confirmed_observation") is not None, "measured_ev_soc_required") + entities = options["entity_config"] + relay = _cached(base, entities["relay_entity"]) + permission = _cached(base, entities["permission_entity"]) + closed_confirmation = options["house_only"] and bool(options.get("confirmed_observation")) + connection = None if closed_confirmation else _cached(base, entities["connection_entity"]) + allowed = entities["connection_entity_states"] + if recurring_enabled(settings): + observation = options.get("confirmed_observation") or {} + acknowledged = observation.get("connection_error_acknowledged") is True and connection is not None and connection["state"] == "4" and connection["last_updated"] == _stamp(observation["connection_error_observed_at"]) + else: + acknowledged = _acknowledged_stop_state(settings, connection, _clock()) + _require(closed_confirmation or acknowledged or (isinstance(allowed, (list, tuple)) and bool(allowed) and connection["state"] in allowed), "connection_not_confirmed") + options["connection_error_acknowledged"] = acknowledged + _require(permission["state"] == "on", "permission_not_granted") + _require(relay["state"] in ("on", "off"), "relay_state_invalid") + now = _clock() + _require(all(payload["last_changed"] <= now and payload["last_updated"] <= now for payload in (relay, permission, connection) if payload is not None), "entity_timestamp_future") + _require(_finite(base.soc_max, 0.001) and _finite(base.soc_kw, base.reserve, base.soc_max), "domestic_soc_invalid") + minute = (now - base.midnight_utc).total_seconds() / 60 + floor = max(base.reserve, base.all_active_keep.get(int(minute), 0) * base.soc_max / 100) + _require(base.soc_kw >= floor - 0.002, "domestic_manual_floor") + _control_fingerprint(inverter) + return options, inverter, relay, now, minute + + +def _solar_observation(base, options, relay, now): + """Require fresh local surplus at full EV power, reserving measured household demand first.""" + inverter = base.inverters[0] + _require(getattr(inverter, "inv_num_load_entities", None) == 1 and not base.get_arg("load_power_invert", False, index=0), "solar_meter_topology_unsupported") + observations = [] + + def power(entity): + """Read explicit watt units from cached entities without refreshing hardware.""" + observed = _cached(base, entity) + age = (now - observed["last_updated"]).total_seconds() + _require(0 <= age < SOLAR_OBSERVATION_SECONDS and observed["last_changed"] <= now, "solar_power_stale_or_future") + unit = observed["attributes"].get("unit_of_measurement") + _require(unit in ("W", "kW") and not isinstance(observed["state"], bool), "solar_power_units_invalid") + try: + value = float(observed["state"]) * (1000 if unit == "kW" else 1) + except (TypeError, ValueError): + raise JointBlocked("solar_power_invalid") from None + _require(_finite(value), "solar_power_invalid") + observations.append(observed["last_updated"]) + return value + + pv = power(base.get_arg("pv_power", None, indirect=False, index=0)) + load = power(base.get_arg("load_power", None, indirect=False, index=0)) + ev_power = 0 + if relay["state"] == "on": + metering = options["settings"].get("recurring_session") or options["settings"].get("confirmed_session") or {} + ev_power = power(metering.get("power_entity")) + _require(ev_power <= load + 200 and ev_power <= base.car_charging_rate[0] * 1100, "solar_power_inconsistent") + _require((max(observations) - min(observations)).total_seconds() <= 30, "solar_power_samples_misaligned") + pv_ac = pv * (base.inverter_loss if base.inverter_hybrid else 1) + if not base.inverter_hybrid and base.pv_ac_limit > 0: + pv_ac = min(pv_ac, base.pv_ac_limit * MINUTE_WATT) + pv_ac = min(pv_ac, base.inverter_limit * MINUTE_WATT) + house = max(0, load - ev_power) + surplus = pv_ac - house + required = base.car_charging_rate[0] * 1000 + (200 if relay["state"] == "off" else 0) + _require(surplus >= required, "solar_surplus_insufficient") + return {"observed_at": min(observations).isoformat(), "surplus_w": surplus, "required_w": required, "household_load_w": house, "pv_ac_w": pv_ac, "ev_metered_w": ev_power} + + +def _ev_permitted(base, options, relay, now, on): + """Check hysteresis and exact elapsed OFF seconds independently of battery mode.""" + if not on: + return + _require(not options["departure_passed"], "departure_passed") + _require(base.soc_kw > options["pause"] * base.soc_max / 100, "ev_pause_threshold") + _require(options["ev_soc"] < base.car_charging_limit[0], "ev_target_already_reached") + session = options["settings"]["session"] + _require(_stamp(session["available_from"]) <= now < _stamp(session["departure"]), "ev_outside_session") + if relay["state"] == "off": + _require(base.soc_kw >= options["restart"] * base.soc_max / 100, "ev_restart_threshold") + _require((now - relay["last_changed"]).total_seconds() >= options["off_minutes"] * 60, "ev_cooldown") + if options["ev_charging_priority"] == "solar": + return _solar_observation(base, options, relay, now) + + +def _intent(command, inverter, charge_ceiling=False): + """Validate exact bounded commands; do not clamp a rejected plan into another plan.""" + _require(isinstance(command, dict), "invalid_command") + target = command.get("target_soc") + bounded = charge_ceiling and getattr(inverter, "inverter_type", None) == "DeyeSolarman" and type(target) is int and math.ceil(inverter.reserve_percent) < target <= 100 + _require(type(target) is int and (target in (0, 100) or bounded), "invalid_target") + _require(type(command.get("freeze")) is bool and type(command.get("ev_on")) is bool, "invalid_command_boolean") + _require(not command["freeze"] or command["target_soc"] == 0, "invalid_hold_target") + if getattr(inverter, "inverter_type", None) == "DeyeSolarman": + _require(command["freeze"] is False and command.get("power_w") is None, "unsupported_template_command") + return command["target_soc"], False, None + cap = min(inverter.deye_local.settings["max_discharge_power_w"], inverter.battery_rate_max_discharge * MINUTE_WATT) + power = command.get("power_w") + _require(_finite(cap, 1), "invalid_discharge_cap") + _require(power is None or (type(power) is int and _finite(power, 0, cap)), "discharge_power_out_of_range") + return command["target_soc"], command["freeze"], power + + +def _candidate(base, pending, inverter, now, minute): + """Validate complete replay and select only the current first planned command.""" + from joint_site_native import _ev_charging_priority + + _require(isinstance(pending, dict), "no_pending_candidate") + report = pending["report"] + partial = report.get("status") == "partial_candidate" + _require(report.get("status") in ("candidate", "partial_candidate") and not report.get("counterfactual_controls") and not report.get("execution_blockers"), "candidate_not_accepted") + if partial: + from joint_site_session import partial_metadata_valid, partial_fingerprint + + _require(partial_metadata_valid(base, report), "partial_target_not_authorised") + _require(pending.get("partial_fingerprint") == partial_fingerprint(report), "partial_evidence_changed") + else: + _require(not report.get("partial_target_authorised") and (report.get("planning_ev_target_kwh") in (None, base.car_charging_limit[0])), "partial_target_not_authorised") + checks = report.get("native_checks", {}) + _require(set(checks) == {"nominal", "pv10"}, "native_checks_missing") + _require(all(check.get("passed") is True and check.get("checked_minutes") == base.forecast_minutes for check in checks.values()), "native_checks_failed") + _require(0 <= (now - _stamp(report["generated_at"])).total_seconds() <= RECEIPT_TTL_SECONDS, "candidate_stale_or_future") + _require(pending.get("fingerprint") == execution_fingerprint(base), "configuration_changed") + _require(pending["options"]["ev_charging_priority"] == _ev_charging_priority(base), "charging_priority_changed_during_planning") + closed = now >= _stamp(pending["options"]["settings"]["session"]["departure"]) + _require(report.get("session_closed", False) is closed, "session_phase_changed") + _require(_finite(pending.get("initial_battery_soc_kwh"), base.reserve, base.soc_max), "initial_domestic_soc_missing") + soc_fell = base.soc_kw < pending["initial_battery_soc_kwh"] - 0.002 + shared_support = bool(report.get("grid_charge_support_w")) + _require(not soc_fell or shared_support, "domestic_soc_fell_since_planning") + commands = pending["commands"] + if shared_support: + _require(pending.get("command_fingerprint") == command_fingerprint(commands), "candidate_commands_changed") + _require(isinstance(commands, list) and len(commands) == base.forecast_minutes // PREDICT_STEP, "incomplete_command_horizon") + for index, command in enumerate(commands): + _intent(command, inverter, bool(report.get("grid_charge_support_w"))) + _require(not closed or not command["ev_on"], "closed_session_ev_command") + _require(type(command.get("start")) is int and type(command.get("end")) is int, "invalid_command_time") + _require(command.get("start") == base.minutes_now + index * PREDICT_STEP and command.get("end") == command["start"] + PREDICT_STEP, "command_timeline_mismatch") + command = commands[0] + _require(command["start"] <= minute < command["end"], "current_command_expired") + if pending["options"]["ev_charging_priority"] == "now" and not closed and pending["options"]["ev_soc"] < base.car_charging_limit[0] - 0.002: + _require(command["ev_on"], "immediate_start_not_validated") + if pending["options"]["ev_charging_priority"] == "solar" and command["ev_on"]: + _require(command["target_soc"] == 0 and not command["freeze"], "solar_requires_demand") + if soc_fell: + # The ordinary native loop re-reads SOC after calculating. A lower value + # cannot be waived with a tolerance: replay the SAME controls from that + # value and require both complete forecasts again, without a new search. + from joint_site_native import _options, _replay, _publish_report + + _require("replay_inputs" in pending, "replay_inputs_missing") + result, slots, windows = pending["replay_inputs"] + options = _options(base) + model = base + if partial: + model = copy(base) + model.car_charging_limit = list(base.car_charging_limit) + model.car_charging_limit[0] = report["planning_ev_target_kwh"] + checks = {name: _replay(model, options, result, slots, windows, scenario) for name, scenario in (("nominal", 0), ("pv10", 1))} + for check in checks.values(): + check.pop("_charge_peak_kwh", None) + updated = {**deepcopy(base.joint_site_report), "native_checks": checks, "dispatch_soc_revalidated_kwh": base.soc_kw} + _publish_report(base, updated, False) + _require(all(check["passed"] and check["checked_minutes"] == base.forecast_minutes for check in checks.values()), "dispatch_replay_failed") + # Only extend an accepted intent over consecutive identical actions; never + # authorise a future start or a different power ceiling before dispatch. + run = deepcopy(command) + for later in commands[1:]: + if any(later[key] != command[key] for key in ("target_soc", "freeze", "power_w", "ev_on")): + break + run["end"] = later["end"] + return command, run + + +def _publish_ev(base, command=None, now=None, expires=None): + """Publish only a currently authorised EV interval, without switching the relay.""" + windows = [] + if command and command["ev_on"]: + start = (now - base.midnight_utc).total_seconds() / 60 + end = (expires - base.midnight_utc).total_seconds() / 60 + if end > start: + rate = base.rate_import[int(start)] + energy = base.car_charging_rate[0] * (end - start) / 60 + windows = [{"start": start, "end": end, "kwh": energy, "average": rate, "cost": rate * energy, "octopus": False}] + base.car_charging_slots = [windows] + [[] for _ in range(PREDBAT_MAX_CARS - 1)] + base.dashboard_item( + "binary_sensor." + base.prefix + "_car_charging_slot", + state="on" if windows else "off", + attributes={ + "planned": windows, + "source": "joint_site_apply", + "ready_to_activate": False, + "hardware_confirmed": False, + "lease_version": 1, + "valid_from": now.isoformat() if windows else None, + "valid_until": expires.isoformat() if windows else None, + }, + ) + base.dashboard_item( + base.prefix + ".car_charging_start", + state=now.strftime("%H:%M:%S") if windows else "", + attributes={"timestamp": now.isoformat() if windows else None, "source": "joint_site_apply", "minutes_to": 0 if windows else base.forecast_minutes}, + ) + + +def _report(base, status, enabled, reason=None, attempted=False): + """Keep model output and observed execution distinct, never claim electrical enforcement.""" + report = deepcopy(getattr(base, "joint_site_report", None) or {}) + report.update(status=status, execution_enabled=enabled, ready_to_activate=False, hardware_confirmed=False, enforced=False, actuator_signals_modified=attempted, execution_reason=reason) + receipt = getattr(base, "joint_site_execution", None) + report["execution_observed_at"] = receipt.get("generated_at") if receipt else None + report["execution_valid_until"] = receipt.get("expires_at") if receipt else None + base.joint_site_report = report + base.dashboard_item(base.prefix + ".joint_site_plan", state=status, attributes={key: value for key, value in report.items() if key != "preview"}) + + +def _blocked(base, reason, attempted=False): + """Revoke previous EV authority on every error, including publication failure.""" + base.joint_site_execution = None + base.inverter_rate_intent = {} + base.isCharging = base.isExporting = False + base.car_charging_slots = [[] for _ in range(PREDBAT_MAX_CARS)] + try: + _publish_ev(base) + _report(base, "apply_blocked", False, reason, attempted) + except Exception: + # If publication itself is down, the existing EV executor must use its + # independent native-status freshness watchdog; do not invent success. + base.joint_site_report = {"status": "apply_blocked", "execution_enabled": False, "ready_to_activate": False, "hardware_confirmed": False, "execution_reason": reason, "publication_failed": True} + try: + base.record_status("Error: Joint apply blocked ({})".format(reason), had_errors=True) + except Exception: + pass + return "Joint blocked", reason + + +def _withdraw_for_household(base, reason, attempted=False): + """Withdraw EV authority, then require observed relay OFF before the sole native handback.""" + base.joint_site_execution = None + base.joint_site_pending = None + base.car_charging_slots = [[] for _ in range(PREDBAT_MAX_CARS)] + try: + _publish_ev(base) + relay = _cached(base, base.args["joint_site"].get("relay_entity")) + now = _clock() + off = relay["state"] == "off" and relay["last_changed"] <= now and relay["last_updated"] <= now + _report(base, "household_fallback" if off else "household_waiting", False, reason, attempted) + base.joint_site_report.update(household_executor="native" if off else "waiting_for_ev_off", ev_available=False) + if off: + base.inverter_rate_intent = {} + return None + return "Joint waiting", "EV authority withdrawn; waiting for observed relay OFF" + except Exception: + # Missing relay or failed publication is not an OFF observation. Existing + # battery support must remain until the external executor is observed OFF. + return _blocked(base, "household_handback_unconfirmed", attempted) + + +def _ev_handoff_until(base, options, run, commands): + """Bound overlap at a domestic-only boundary inside an already accepted EV-ON run. + + The ordinary five-minute calculation finishes just after the boundary. Do not + turn that dispatch latency into an EV stop and ten-minute restart cooldown. + This does not dispatch a future domestic intent or permit a future EV start. + """ + if options["control_model"] != "deye_solarman" or not options.get("grid_charge_support_w") or not options.get("confirmed_observation") or not run["ev_on"]: + return None + following = next((item for item in commands if item["start"] == run["end"]), None) + if following is None or not following["ev_on"] or any(following[key] != run[key] for key in ("freeze", "power_w")): + return None + return (base.midnight_utc + timedelta(minutes=run["end"], seconds=EV_HANDOFF_SECONDS)).isoformat() + + +def _receipt_authority_end(base, options, receipt): + """Keep overlap non-rolling and independently capped, while retaining every live guard.""" + command = receipt["command"] + end = base.midnight_utc + timedelta(minutes=command["end"]) + handoff = receipt.get("ev_handoff_until") + if handoff is None: + return end + handoff = _stamp(handoff) + _require(options["control_model"] == "deye_solarman" and options.get("grid_charge_support_w") and options.get("confirmed_observation") and command["ev_on"], "ev_handoff_not_supported") + _require(0 < (handoff - end).total_seconds() <= EV_HANDOFF_SECONDS, "ev_handoff_out_of_bounds") + return handoff + + +def execute_joint_site(base): + """Execute joint intents or hand back only after revocation and observed EV OFF.""" + if not _requested(base): + return None + attempted = False + try: + from joint_site_session import household_reason + + reason = household_reason(base) + if reason: + return _withdraw_for_household(base, reason) + options, inverter, relay, now, minute = _evidence(base, allow_read_only_transition=True) + read_only_transition = bool(getattr(base, "inverter_needs_reset", False)) + pending = getattr(base, "joint_site_pending", None) + command, run = _candidate(base, pending, inverter, now, minute) + if options.get("grid_charge_support_w"): + _require(abs(options["ev_soc"] - pending["options"]["ev_soc"]) <= 0.002, "dispatch_ev_soc_changed") + _ev_permitted(base, options, relay, now, command["ev_on"]) + _require(not read_only_transition or relay["state"] == "off", "waiting_for_ev_relay_off") + if not command["ev_on"] and relay["state"] == "on": + # Remove EV permission first, then wait for the sole EV executor's + # actual OFF observation before withdrawing battery support. + return _blocked(base, "waiting_for_ev_relay_off") + target, freeze, power = _intent(command, inverter, bool(options.get("grid_charge_support_w"))) + base.inverter_rate_intent = {} + # No call to the adapter directly: this is the same native entry point + # whose optional DEYE implementation owns all physical service writes. + attempted = True + if options["control_model"] == "deye_solarman": + from joint_site_template import dispatch_template + + if options.get("grid_charge_support_w"): + observed = dispatch_template(inverter, target, charge_ceiling=True) + else: + observed = dispatch_template(inverter, target) + else: + observed = inverter.adjust_charge_immediate(target, freeze=freeze, discharge_power_w=power) + _require(observed is True, "inverter_command_not_observed") + # Permission, clocks and protection inputs can change while writes settle. + options, inverter, relay, now, minute = _evidence(base, allow_read_only_transition=True) + _require(pending["fingerprint"] == execution_fingerprint(base), "configuration_changed_during_dispatch") + _require(pending["report"].get("session_closed", False) is options["departure_passed"], "session_phase_changed") + _require(command["start"] <= minute < command["end"], "command_expired_during_dispatch") + solar = _ev_permitted(base, options, relay, now, command["ev_on"]) + if read_only_transition: + # The native read-only switch requests reinitialisation. The shared + # executor fulfils it by observing ALL canonical rows of this exact + # paired-safe intent, rather than running the unrelated native reset + # (which can rewrite standing power/reserve settings). Failures above + # leave the request pending; other reset reasons remain unsupported. + _require(base.inverter_needs_reset and base.inverter_needs_reset_force == "set_read_only", "reset_request_changed") + base.inverter_needs_reset = False + base.inverter_needs_reset_force = "" + handoff = _ev_handoff_until(base, options, run, pending["commands"]) + authority_end = _stamp(handoff) if handoff else base.midnight_utc + timedelta(minutes=run["end"]) + expires = min(authority_end, now + timedelta(seconds=RECEIPT_TTL_SECONDS)) + if command["ev_on"]: + expires = min(expires, _stamp(options["settings"]["session"]["departure"])) + if solar: + expires = min(expires, now + timedelta(seconds=SOLAR_RECEIPT_SECONDS), _stamp(solar["observed_at"]) + timedelta(seconds=SOLAR_OBSERVATION_SECONDS)) + base.joint_site_execution = { + "generated_at": now.isoformat(), + "plan_generated_at": pending["report"]["generated_at"], + "expires_at": expires.isoformat(), + "command": run, + "ev_handoff_until": handoff, + "fingerprint": execution_fingerprint(base), + "control_fingerprint": _control_fingerprint(inverter), + "observations_match": True, + "hardware_confirmed": False, + "connection_error_acknowledged": options.get("connection_error_acknowledged", False), + "read_only_transition_observed": read_only_transition, + "solar_observation": solar, + } + base.isCharging = target > 0 or freeze + base.isExporting = False + _publish_ev(base, run, now, expires) + _report(base, "applied", True, "inverter_observed_not_hardware_confirmed", attempted=True) + return "Freeze charging" if freeze else "Charging" if target > 0 else "Demand", "Joint site; HA observations only" + except Exception as exc: + reason = str(exc) if isinstance(exc, JointBlocked) else "execution_validation_failed" + if base.args["joint_site"].get("household_fallback") is True: + return _withdraw_for_household(base, reason, attempted) + return _blocked(base, reason, attempted) + + +def publish_joint_ev(base, renew=False): + """Prevent early native EV publication; revalidate a bounded receipt without actuating.""" + if not _requested(base): + return False + receipt = getattr(base, "joint_site_execution", None) + try: + from joint_site_session import household_reason + + reason = household_reason(base) + if reason: + _withdraw_for_household(base, reason) + return True + _require(isinstance(receipt, dict) and receipt.get("observations_match") is True, "no_execution_receipt") + options, inverter, relay, now, minute = _evidence(base) + _require(receipt["fingerprint"] == execution_fingerprint(base), "configuration_changed") + _require(receipt["control_fingerprint"] == _control_fingerprint(inverter), "observed_controls_changed") + _require(0 <= (now - _stamp(receipt["generated_at"])).total_seconds() < RECEIPT_TTL_SECONDS, "receipt_expired") + expires = _stamp(receipt["expires_at"]) + _require(now < expires, "command_authority_expired") + command = receipt["command"] + _intent(command, inverter, bool(options.get("grid_charge_support_w"))) + authority_end = _receipt_authority_end(base, options, receipt) + _require(command["start"] <= minute and now < authority_end, "command_authority_expired") + solar = _ev_permitted(base, options, relay, now, command["ev_on"]) + if solar: + _require(command["target_soc"] == 0 and not command["freeze"], "solar_requires_demand") + if renew: + # Revalidate only the already observed, identical current intent. + # The immutable plan clock bounds renewal across one delayed calculation; + # it must never become an indefinitely rolling lease on an old forecast. + _require(options.get("grid_charge_support_w") and options.get("confirmed_observation"), "lease_renewal_not_supported") + plan_time = _stamp(receipt["plan_generated_at"]) + _require(0 <= (now - plan_time).total_seconds() < PLAN_REVALIDATION_MAX_AGE_SECONDS, "plan_revalidation_expired") + expires = min(authority_end, now + timedelta(seconds=RECEIPT_TTL_SECONDS), plan_time + timedelta(seconds=PLAN_REVALIDATION_MAX_AGE_SECONDS)) + if command["ev_on"]: + expires = min(expires, _stamp(options["settings"]["session"]["departure"])) + if solar: + expires = min(expires, now + timedelta(seconds=SOLAR_RECEIPT_SECONDS), _stamp(solar["observed_at"]) + timedelta(seconds=SOLAR_OBSERVATION_SECONDS)) + receipt = {**receipt, "generated_at": now.isoformat(), "expires_at": expires.isoformat()} + if solar: + receipt["solar_observation"] = solar + base.joint_site_execution = receipt + _publish_ev(base, command, now, expires) + if renew: + _report(base, "applied", True, "inverter_observed_not_hardware_confirmed", attempted=True) + except Exception as exc: + # Fetch-time absence of a receipt is normal at startup, not proof the + # native planner is broken. It still revokes any old EV-slot signal. + base.joint_site_execution = None + base.car_charging_slots = [[] for _ in range(PREDBAT_MAX_CARS)] + try: + _publish_ev(base) + if receipt: + _report(base, "apply_blocked", False, str(exc) if isinstance(exc, JointBlocked) else "execution_validation_failed") + except Exception: + _blocked(base, str(exc) if isinstance(exc, JointBlocked) else "publication_failed") + return True + + +def refresh_joint_ev_lease(base): + """Use the existing cache-only timer to sustain an accepted intent, never to start a new one.""" + settings = base.args.get("joint_site") + from joint_site_recurring import recurring_enabled + + if _requested(base) and settings.get("grid_charge_support") and (settings.get("confirmed_session") or recurring_enabled(settings)) and getattr(base, "joint_site_execution", None): + publish_joint_ev(base, renew=True) diff --git a/apps/predbat/joint_site_arrival.py b/apps/predbat/joint_site_arrival.py new file mode 100644 index 000000000..ec3df551b --- /dev/null +++ b/apps/predbat/joint_site_arrival.py @@ -0,0 +1,222 @@ +"""Conditional absent-EV demand for native prediction, never a charger plan or permission.""" + +from copy import deepcopy +from datetime import datetime, timedelta, timezone +import hashlib +import json +import math +from zoneinfo import ZoneInfo + +from const import PREDICT_STEP +from joint_site_recurring import next_departure, recurring_enabled, resolve_recurring_session + + +def _signature(base): + """Invalidate cached assumptions when the user changes policy, target, rate or ready time.""" + values = [base.args.get("joint_site"), base.args.get("timezone"), base.car_charging_limit[0], base.car_charging_battery_size[0], base.car_charging_rate[0], base.car_charging_loss, base.car_charging_plan_time[0], base.forecast_minutes] + return hashlib.sha256(json.dumps(values, sort_keys=True, allow_nan=False).encode()).hexdigest() + + +def _eligible(base, now): + """Check absence and idle authority first, without reading any vehicle SOC or energy.""" + from joint_site_native import _read_entity_state + + settings = base.args.get("joint_site") + if not recurring_enabled(settings) or settings.get("execution_authorised") is not True or settings.get("household_fallback") is not True: + return False + if base.num_cars != 1 or base.car_charging_slots[0] or base.car_charging_planned[0] or base.car_charging_now_active(0) or base.car_charging_manual_soc[0] or base.car_charging_soc[0] != 0: + return False + # Native SOC zero is the recurring path's inactive placeholder, NOT a reading. + for key, state in (("home_entity", "not_home"), ("permission_entity", "on"), ("relay_entity", "off")): + item = _read_entity_state(base, settings.get(key)) + if item is None or item["state"] != state or max(item["last_changed"], item["last_updated"]) > now: + return False + return True + + +def _trips(history, now): + """Use paired real home/away transitions; force updates and unknown gaps create no trips.""" + from joint_site_native import _parse_entity_timestamp + + if not isinstance(history, list) or len(history) != 1 or not isinstance(history[0], list) or len(history[0]) > 50000: + raise ValueError("arrival_history_invalid") + records = [] + for item in history[0]: + stamp = _parse_entity_timestamp(item["last_changed"], "arrival_history_time") + if stamp > now: + raise ValueError("arrival_history_future") + if stamp >= now - timedelta(days=14): + records.append((stamp, item["state"])) + previous = None + previous_at = None + left = None + trips = [] + for stamp, state in sorted(records): + if previous_at == stamp and previous != state: + raise ValueError("arrival_history_ambiguous") + if state == previous: + previous_at = stamp + continue + if state == "not_home" and previous == "home": + left = stamp + elif state == "home" and previous == "not_home" and left is not None: + duration = (stamp - left).total_seconds() + if 60 <= duration <= 86400: + trips.append((left, duration)) + left = None + else: + left = None + previous, previous_at = state, stamp + return trips, left if previous == "not_home" else None + + +def _quantile(values, fraction): + """Interpolate a descriptive empirical percentile, not a guaranteed arrival bound.""" + values = sorted(values) + position = (len(values) - 1) * fraction + lower = int(position) + upper = min(lower + 1, len(values) - 1) + return values[lower] + (values[upper] - values[lower]) * (position - lower) + + +def _arrival_samples(trips, left, now, zone, deadline): + """Condition on trip duration, falling back to genuine past daily return clocks when sparse.""" + local = left.astimezone(zone) + elapsed = (now - left).total_seconds() + surviving = [(start, duration) for start, duration in trips if duration > elapsed] + if len(surviving) >= 3: + similar = [(start, duration) for start, duration in surviving if abs((start.astimezone(zone).hour - local.hour + 12) % 24 - 12) <= 3] + cohort = "all_remaining_trips" + if len(similar) >= 3: + surviving, cohort = similar, "similar_departure_hours" + same_day_type = [(start, duration) for start, duration in similar if (start.astimezone(zone).weekday() < 5) == (local.weekday() < 5)] + if len(same_day_type) >= 3: + surviving, cohort = same_day_type, "similar_hours_and_day_type" + return [(left + timedelta(seconds=duration) - now).total_seconds() for start, duration in surviving], cohort + # Long trips can outlast most short-trip examples. Do not turn a sparse tail + # into a precise ETA: use one last genuine return per completed local day. + last_returns = {} + today = now.astimezone(zone).date() + for start, duration in trips: + returned = (start + timedelta(seconds=duration)).astimezone(zone) + if returned.date() < today: + last_returns[returned.date()] = max(returned, last_returns.get(returned.date(), returned)) + arrivals = [] + for returned in last_returns.values(): + candidate = datetime.combine(today, returned.time(), tzinfo=zone) + if candidate <= now.astimezone(zone): + candidate = datetime.combine(today + timedelta(days=1), returned.time(), tzinfo=zone) + remaining = (candidate.astimezone(timezone.utc) - now).total_seconds() + if 0 < remaining < (deadline - now).total_seconds(): + arrivals.append(remaining) + if len(arrivals) < 3: + raise ValueError("arrival_insufficient_remaining_history") + return arrivals, "historical_last_return_clock" + + +def _prepare(base, now): + """Build one low-confidence, empty-car demand scenario from cached recorder history.""" + from ha import run_async + from joint_site_native import _number, _read_entity_state + + entity = base.args["joint_site"]["home_entity"] + # The same authority validator must accept the standing policy, but still + # reject actual EV availability. A retained fault cannot forecast a charge. + if resolve_recurring_session(base, now=now).get("reason") != "ev_not_home": + raise ValueError("arrival_policy_invalid") + record = run_async(base.components.get_component("storage").load_checked("joint_site", "recurring_session")) + if record is None or record["invalid"]: + raise ValueError("arrival_policy_fault") + home_transition = _read_entity_state(base, entity)["value_last_changed"].isoformat() + cached = getattr(base, "joint_site_arrival_history", None) + if not cached or cached["entity"] != entity or cached["home_transition"] != home_transition or not 0 <= (now - cached["at"]).total_seconds() < 3600: + cached = {"entity": entity, "home_transition": home_transition, "at": now, "history": base.get_history_wrapper(entity, days=14, required=False)} + base.joint_site_arrival_history = cached + trips, left = _trips(cached["history"], now) + latest_history_state = max((datetime.fromisoformat(item["last_changed"]) for item in cached["history"][0]), default=None) + if latest_history_state is None or latest_history_state < datetime.fromisoformat(home_transition): + base.joint_site_arrival_history = None + raise ValueError("arrival_history_does_not_cover_departure") + if left is None: + raise ValueError("arrival_departure_not_observed") + zone = ZoneInfo(base.args["timezone"]) + deadline = next_departure(base, now) + remaining, cohort = _arrival_samples(trips, left, now, zone, deadline) + expected = now + timedelta(seconds=_quantile(remaining, 0.75)) + low, high = (now + timedelta(seconds=_quantile(remaining, fraction)) for fraction in (0.1, 0.9)) + midnight = base.midnight_utc.astimezone(timezone.utc) + start = math.ceil((max(expected, now + timedelta(minutes=PREDICT_STEP)) - midnight).total_seconds() / (60 * PREDICT_STEP)) * PREDICT_STEP + capacity = _number(base.car_charging_battery_size[0], "arrival_capacity", 0.001) + target = _number(base.car_charging_limit[0], "arrival_target", 0, capacity) + rate = _number(base.car_charging_rate[0], "arrival_rate", 0.001) + efficiency = _number(base.car_charging_loss, "arrival_efficiency", 0.001, 1) + horizon = min((deadline - midnight).total_seconds() / 60, base.minutes_now + base.forecast_minutes) + end = min(start + math.ceil(target / efficiency / rate * 60 / PREDICT_STEP) * PREDICT_STEP, math.floor(horizon / PREDICT_STEP) * PREDICT_STEP) + energy = min(target / efficiency, rate * max(0, end - start) / 60) + return { + "status": "anticipated" if energy else "no_remaining_window", + "prepared_at": now.isoformat(), + "expected_arrival": expected.isoformat(), + "arrival_p10": low.isoformat(), + "arrival_p90": high.isoformat(), + "departure_observed_at": left.isoformat(), + "ready_by": deadline.isoformat(), + "completed_trips": len(trips), + "eligible_trips": len(remaining), + "cohort": cohort, + "history_fetched_at": cached["at"].isoformat(), + "latest_history_state_at": latest_history_state.isoformat(), + "confidence": "low", + "soc_assumption_kwh": 0, + "soc_assumption": "empty_car_demand_scenario_not_a_vehicle_reading", + "soc_observed_at": None, + "requested_target_kwh": target, + "model_ac_kwh": energy, + "shortfall_kwh": max(0, target - energy * efficiency), + "guaranteed_target_met": False, + "execution_permission": False, + "requires_observed_home_and_connection": True, + "_signature": _signature(base), + "_plan_at": base.now_utc.isoformat(), + "_home_transition": home_transition, + "_slots": [{"start": start, "end": end, "kwh": energy, "octopus": False}] if energy else [], + } + + +def prepare_arrival_forecast(base): + """Refresh model-only demand once per native fetch; failure leaves household controls native.""" + from joint_site_native import _observation_clock + + base.joint_site_arrival_forecast = None + try: + now = _observation_clock() + if _eligible(base, now): + base.joint_site_arrival_forecast = _prepare(base, now) + except Exception as exc: + reason = str(exc) if isinstance(exc, ValueError) and str(exc).startswith("arrival_") else "arrival_history_or_inputs_unusable" + base.joint_site_arrival_forecast = {"status": "unavailable", "reason": reason, "execution_permission": False} + + +def model_arrival_slots(base): + """Return detached model slots only while all original absence assumptions still hold.""" + from joint_site_native import _observation_clock, _read_entity_state + + report = getattr(base, "joint_site_arrival_forecast", None) + if not isinstance(report, dict) or report.get("status") != "anticipated": + return [] + try: + now = _observation_clock() + if not 0 <= (now - datetime.fromisoformat(report["prepared_at"])).total_seconds() <= 300 or report["_plan_at"] != base.now_utc.isoformat() or report["_signature"] != _signature(base) or not _eligible(base, now): + return [] + home = _read_entity_state(base, base.args["joint_site"]["home_entity"]) + if home["value_last_changed"].isoformat() != report["_home_transition"]: + return [] + return deepcopy(report["_slots"]) + except (ValueError, TypeError, KeyError, AttributeError, IndexError): + return [] + + +def arrival_evidence(base): + """Expose assumptions in Show Debug without projecting them into an execution lease.""" + report = getattr(base, "joint_site_arrival_forecast", None) + return {key: deepcopy(value) for key, value in report.items() if not key.startswith("_")} if isinstance(report, dict) else None diff --git a/apps/predbat/joint_site_native.py b/apps/predbat/joint_site_native.py new file mode 100644 index 000000000..34275e919 --- /dev/null +++ b/apps/predbat/joint_site_native.py @@ -0,0 +1,1382 @@ +"""Opt-in joint scheduling with isolated shadow and separately gated execution. + +The solver makes coupled EV/battery decisions, then a fresh native Prediction +replays them. Shadow never changes executable controls. Explicit apply stages a +candidate for the native executor; staging alone cannot energise an EV or inverter. +""" + +from copy import copy, deepcopy +from datetime import datetime, timezone +import math + +from const import PREDICT_STEP, PREDBAT_MAX_CARS +from prediction import Prediction + + +class UnsupportedJointSite(ValueError): + """A native configuration cannot be represented by this single-EV prototype.""" + + +def _number(value, name, minimum=0, maximum=None): + """Reject coercion, booleans and non-finite configuration/forecast values.""" + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < minimum or (maximum is not None and value > maximum): + raise ValueError("Invalid joint-site numeric field: " + name) + return float(value) + + +def _aware(value, name): + """Require an offset-aware timestamp and reject anything naive; never guess a timezone.""" + if not isinstance(value, datetime): + raise ValueError("Joint-site timestamp required: " + name) + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("Joint-site timestamp needs an offset: " + name) + return value.astimezone(timezone.utc) + + +def _parse_entity_timestamp(value, name): + """Parse an offset-aware ISO 8601 timestamp from HA; never guess a timezone or next day.""" + if not isinstance(value, str): + raise ValueError("Joint-site entity timestamp required: " + name) + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + raise ValueError("Invalid joint-site entity timestamp: " + name) from None + return _aware(parsed, name) + + +def _read_entity_state(base, entity_id): + """Return the raw HA state payload via get_state_wrapper with no refresh. + + Uses the standard HA shape ``{state, attributes, last_changed, last_updated}`` + with ``refresh=False`` so no HA service is ever called. Returns ``None`` when + the interface, the entity or the cache is unavailable. Never invokes a + hardware poll or a cache writer. + """ + if not entity_id or not isinstance(entity_id, str): + return None + try: + payload = base.get_state_wrapper(entity_id=entity_id, raw=True, refresh=False) + except Exception: + return None + if not isinstance(payload, dict): + return None + state = payload.get("state") + last_changed = payload.get("last_changed") + last_updated = payload.get("last_updated") + if state is None or last_changed is None or last_updated is None: + return None + try: + last_changed_aware = _parse_entity_timestamp(last_changed, "last_changed") if isinstance(last_changed, str) else _aware(last_changed, "last_changed") + last_updated_aware = _parse_entity_timestamp(last_updated, "last_updated") if isinstance(last_updated, str) else _aware(last_updated, "last_updated") + except ValueError: + return None + transition = payload.get("value_last_changed") + try: + transition = _parse_entity_timestamp(transition, "value_last_changed") if isinstance(transition, str) else _aware(transition, "value_last_changed") + if transition > last_changed_aware: + transition = last_changed_aware + except ValueError: + transition = last_changed_aware + return { + "state": state, + "last_changed": last_changed_aware, + "last_updated": last_updated_aware, + "value_last_changed": transition, + "value_history": deepcopy(payload.get("value_history")), + "attributes": deepcopy(payload.get("attributes", {})), + } + + +def _parse_soc_raw(state, unit, capacity_kwh): + """Parse an EV SoC raw value honouring the explicit unit and capacity. + + Returns ``None`` when the value is invalid (bool, non-finite, unknown string, + NaN). When the unit is ``%``, the value is multiplied by the capacity. + Numeric strings are parsed exactly once; anything else is rejected so an + invalid reading is never clamped into an apparently valid kWh. + """ + if unit not in ("kWh", "%"): + raise ValueError("soc_entity_unit must be 'kWh' or '%'") + if isinstance(state, bool): + return None + if isinstance(state, (int, float)): + value = float(state) + elif isinstance(state, str): + try: + value = float(state) + except ValueError: + return None + else: + return None + if not math.isfinite(value): + return None + if unit == "%": + return value * capacity_kwh / 100.0 + return value + + +def _observation_clock(): + """Read wall time, never replace an entity's actual observation timestamp.""" + return datetime.now(timezone.utc) + + +def _ev_charging_priority(base): + """Resolve a literal priority or a cached HA dropdown without refreshing devices.""" + settings = base.args.get("joint_site") + priority = settings.get("ev_charging_priority", "cost") if isinstance(settings, dict) else None + if isinstance(priority, str) and priority.startswith(("input_select.", "select.")): + observed = _read_entity_state(base, priority) + return observed["state"] if observed else None + return priority + + +def _options(base): + """Read a dated expected session and entity-driven evidence without treating them as permission to switch.""" + settings = base.args.get("joint_site") + if settings is None or settings == {}: + return None + if not isinstance(settings, dict): + raise ValueError("joint_site must be a mapping") + allowed = { + "mode", + "session", + "load_port_limit_w", + "ev_pause_percent", + "ev_restart_percent", + "min_off_minutes", + "beam_width", + "terminal_min_percent", + "soc_max_age_minutes", + "relay_entity", + "connection_entity", + "permission_entity", + "soc_entity", + "soc_entity_unit", + "connection_entity_states", + "import_limit_w", + "control_model", + "allow_stale_shadow_soc", + "apply_verified", + "ev_executor_lease_verified", + "connection_observed_at_entity", + "soc_observed_at_entity", + "grid_charge_support", + "confirmed_session", + "execution_authorised", + "connection_error_acknowledgement", + "ev_charging_priority", + "self_consumption_first", + "household_fallback", + "home_entity", + "ev_partial_target_authorised", + "recurring_session", + } + if set(settings) - allowed: + raise ValueError("Unknown joint_site option") + mode = settings.get("mode", "off") + if mode == "off": + return None + if mode not in ("shadow", "apply"): + raise ValueError("joint_site mode must be off, shadow or apply") + control_model = settings.get("control_model", "native") + if control_model not in ("native", "deye_tou", "deye_solarman"): + raise ValueError("Unsupported joint-site control model") + ev_priority = _ev_charging_priority(base) + if ev_priority not in ("cost", "asap", "now", "solar"): + raise ValueError("EV charging priority must be cost, asap, now or solar") + if ev_priority in ("asap", "now", "solar") and (control_model != "deye_solarman" or not settings.get("grid_charge_support")): + raise ValueError("ASAP, NOW and SOLAR charging require shared grid-support controls") + self_consumption_first = settings.get("self_consumption_first", False) + if type(self_consumption_first) is not bool: + raise ValueError("self_consumption_first must be boolean") + if self_consumption_first and (control_model != "deye_solarman" or not settings.get("grid_charge_support")): + raise ValueError("Self-consumption preference requires shared grid-support controls") + if type(settings.get("allow_stale_shadow_soc", False)) is not bool: + raise ValueError("allow_stale_shadow_soc must be boolean") + if type(settings.get("apply_verified", False)) is not bool: + raise ValueError("apply_verified must be boolean") + if type(settings.get("ev_executor_lease_verified", False)) is not bool: + raise ValueError("ev_executor_lease_verified must be boolean") + if type(settings.get("execution_authorised", False)) is not bool: + raise ValueError("execution_authorised must be boolean") + for name in ("household_fallback", "ev_partial_target_authorised"): + if type(settings.get(name, False)) is not bool: + raise ValueError("Joint routing and partial authorisation must be boolean") + if settings.get("home_entity") is not None and (not isinstance(settings["home_entity"], str) or not settings["home_entity"].startswith("device_tracker.")): + raise ValueError("home_entity must be a vehicle device tracker") + if mode == "apply" and (control_model not in ("deye_tou", "deye_solarman") or settings.get("allow_stale_shadow_soc", False)): + raise ValueError("Apply requires supported real controls and cannot use stale SOC assumptions") + for name in ("connection_observed_at_entity", "soc_observed_at_entity"): + if settings.get(name) is not None and (not isinstance(settings[name], str) or not settings[name]): + raise ValueError("Observation timestamp entities must be non-empty strings") + _number(base.soc_max, "soc_max", 0.001) + _number(base.reserve, "reserve", 0, base.soc_max) + from joint_site_recurring import recurring_enabled, resolve_recurring_session + + recurring = None + if recurring_enabled(settings): + if "session" in settings or "confirmed_session" in settings: + raise ValueError("Recurring and dated sessions are mutually exclusive") + recurring = resolve_recurring_session(base, settings) + if recurring["status"] != "ready": + raise ValueError("Recurring session is unavailable: " + recurring["reason"]) + settings = {**settings, "session": recurring["session"]} + session = settings.get("session") + if not isinstance(session, dict) or set(session) != {"available_from", "departure", "soc_kwh", "soc_observed_at"}: + raise ValueError("A complete dated joint_site session is required") + now = base.now_utc.astimezone(timezone.utc) + relay_entity = settings.get("relay_entity") + soc_entity = settings.get("soc_entity") + soc_entity_unit = settings.get("soc_entity_unit", "kWh") + connection_entity = settings.get("connection_entity") + permission_entity = settings.get("permission_entity") + connection_entity_states = settings.get("connection_entity_states") + if relay_entity is not None and (not isinstance(relay_entity, str) or not relay_entity): + raise ValueError("relay_entity must be a non-empty string") + if soc_entity is not None and (not isinstance(soc_entity, str) or not soc_entity): + raise ValueError("soc_entity must be a non-empty string") + if connection_entity is not None and (not isinstance(connection_entity, str) or not connection_entity): + raise ValueError("connection_entity must be a non-empty string") + if permission_entity is not None and (not isinstance(permission_entity, str) or not permission_entity): + raise ValueError("permission_entity must be a non-empty string") + if connection_entity_states is not None: + if not isinstance(connection_entity_states, (list, tuple)) or not connection_entity_states or not all(isinstance(item, str) for item in connection_entity_states): + raise ValueError("connection_entity_states must be a non-empty list of strings") + if not isinstance(soc_entity_unit, str) or soc_entity_unit not in ("kWh", "%"): + raise ValueError("soc_entity_unit must be 'kWh' or '%'") + # Live EV SoC retrieval: a configured soc_entity MUST be honoured. A fresh + # entity observation replaces the dated session value before the dated + # observation's age check, so a live reading is never invalidated by a + # stale dated session. A configured but unavailable/invalid entity is a + # blocker, not a fallback. The explicit per-car manual SoC target + # (car_charging_manual_soc_kwh) overrides the live entity so a deliberate + # user setting is never silently replaced. + relay_payload = _read_entity_state(base, relay_entity) if relay_entity else None + soc_payload = _read_entity_state(base, soc_entity) if soc_entity else None + connection_payload = _read_entity_state(base, connection_entity) if connection_entity else None + permission_payload = _read_entity_state(base, permission_entity) if permission_entity else None + # Predbat's forecast reference is floored to five minutes. A real observation + # later in that block is NOT future-dated. Read the observation clock after + # the cached states so a concurrent HA update is compared with current time. + observed_now = _aware(_observation_clock(), "observation_clock") + manual_target_active = bool(getattr(base, "car_charging_manual_soc", None) and base.car_charging_manual_soc and base.car_charging_manual_soc[0]) + manual_target_kwh = None + if manual_target_active: + try: + manual_target_kwh = float(base.car_charging_soc[0]) + except (TypeError, ValueError, IndexError): + manual_target_kwh = None + ev_soc = None + ev_soc_source = None + stale_soc = None + soc_observed_at = None + soc_age_minutes = None + entity_blockers = [] + support_w = 0.0 + support = settings.get("grid_charge_support") + if support is not None: + if control_model != "deye_solarman" or not isinstance(support, dict) or set(support) != {"enabled_entity", "limit_entity"}: + raise ValueError("Grid charge support requires explicit cached entities and shared controls") + if any(not isinstance(value, str) or not value for value in support.values()): + raise ValueError("Grid charge support entities must be non-empty strings") + enabled = _read_entity_state(base, support["enabled_entity"]) + limit = _read_entity_state(base, support["limit_entity"]) + try: + if enabled is None or enabled["state"] != "on" or limit is None or isinstance(limit["state"], bool): + raise ValueError("Grid support unavailable") + support_w = _number(float(limit["state"]), "grid_charge_support_w", 1, _number(settings.get("import_limit_w", getattr(base, "import_limit", 0)), "import_limit_w", 1)) + if enabled["last_updated"] > observed_now or limit["last_updated"] > observed_now: + raise ValueError("Future grid support observation") + except (TypeError, ValueError): + support_w = 0.0 + entity_blockers.append("grid_charge_support_unavailable") + confirmed_observation = recurring["observation"] if recurring else None + if settings.get("confirmed_session") is not None: + from joint_site_observation import confirmed_session_observation + + if control_model != "deye_solarman" or manual_target_active: + raise ValueError("Confirmed session cannot override a manual SOC target") + confirmed_observation = confirmed_session_observation(base, settings, observed_now) + if mode == "apply" and not confirmed_observation and (manual_target_active or soc_entity is None): + # The native manual SOC can advance from prediction; it is not a fresh + # vehicle observation, even when another vehicle timestamp is recent. + entity_blockers.append("measured_ev_soc_required") + if confirmed_observation: + ev_soc = confirmed_observation["ev_soc"] + ev_soc_source = confirmed_observation["ev_soc_source"] + soc_observed_at = confirmed_observation["soc_observed_at"] + soc_age_minutes = confirmed_observation["soc_observation_age_minutes"] + elif manual_target_kwh is not None: + ev_soc = _number(manual_target_kwh, "manual_ev_soc", 0, base.car_charging_battery_size[0]) + ev_soc_source = "manual_soc_target" + elif soc_entity is None: + # No live entity configured: dated session is the only source. + observed = _aware(session["soc_observed_at"], "soc_observed_at") if isinstance(session["soc_observed_at"], datetime) else _parse_entity_timestamp(session["soc_observed_at"], "soc_observed_at") + max_age = _number(settings.get("soc_max_age_minutes", 60), "soc_max_age_minutes", 1, 1440) + age = (observed_now - observed).total_seconds() / 60 + if age < 0 or age > max_age: + raise ValueError("EV SoC observation is stale or future-dated") + ev_soc = _number(session["soc_kwh"], "soc_kwh", 0, base.car_charging_battery_size[0]) + ev_soc_source = "dated_session" + else: + # Live entity configured: a fresh observation is the only valid source. + # The dated session is NOT a fallback; absence, invalid value or stale + # timestamp all become structured blockers. + if soc_payload is None: + entity_blockers.append("soc_entity_unavailable") + else: + ev_soc_kwh = _parse_soc_raw(soc_payload["state"], soc_entity_unit, base.car_charging_battery_size[0]) + if ev_soc_kwh is None: + entity_blockers.append("soc_entity_invalid_value") + elif ev_soc_kwh < 0 or ev_soc_kwh > base.car_charging_battery_size[0]: + entity_blockers.append("soc_entity_out_of_bounds") + else: + soc_max_age = _number(settings.get("soc_max_age_minutes", 60), "soc_max_age_minutes", 1, 1440) + observed = soc_payload["last_updated"] + if mode == "apply": + # Unchanged values need not change HA last_updated. Apply + # requires the independent field-specific observation time. + stamp_payload = _read_entity_state(base, settings.get("soc_observed_at_entity")) + if stamp_payload is None: + raise ValueError("Apply requires a physical SOC observation timestamp") + observed = _parse_entity_timestamp(stamp_payload["state"], "physical_soc_observation") + soc_max_age = min(soc_max_age, 3) + if soc_payload["last_updated"] > observed_now: + entity_blockers.append("soc_entity_timestamp_future") + soc_age_minutes = (observed_now - observed).total_seconds() / 60 + soc_observed_at = observed.isoformat() + if soc_age_minutes < 0 or soc_age_minutes > soc_max_age: + entity_blockers.append("soc_entity_stale_or_future") + if soc_age_minutes > soc_max_age: + stale_soc = ev_soc_kwh + else: + ev_soc = ev_soc_kwh + ev_soc_source = "entity_cache" + arrival = _aware(session["available_from"], "available_from") if isinstance(session["available_from"], datetime) else _parse_entity_timestamp(session["available_from"], "available_from") + departure = _aware(session["departure"], "departure") if isinstance(session["departure"], datetime) else _parse_entity_timestamp(session["departure"], "departure") + if arrival >= departure or (departure - now).total_seconds() > base.forecast_minutes * 60: + raise ValueError("Expected EV session is outside the forecast/departure window") + start_offset = max(math.ceil((arrival - now).total_seconds() / (60 * PREDICT_STEP)) * PREDICT_STEP, 0) + end_offset = max(0, math.floor((departure - now).total_seconds() / (60 * PREDICT_STEP)) * PREDICT_STEP) + pause = _number(settings.get("ev_pause_percent", 15), "ev_pause_percent", 0, 100) + restart = _number(settings.get("ev_restart_percent", 25), "ev_restart_percent", pause, 100) + off_minutes = _number(settings.get("min_off_minutes", 10), "min_off_minutes", PREDICT_STEP, 60) + beam = _number(settings.get("beam_width", 128), "beam_width", 8, 512) + if beam != int(beam) or off_minutes != int(off_minutes): + raise ValueError("Beam width and cooldown must be integers") + # Relay-driven initial state: only the relay entity defines the physical + # switch state. The state value ('on'/'off') is read explicitly so an 'off' + # can never masquerade as 'on'. The relay's last_changed defines the + # elapsed off time; rounding DOWN means the cooldown remaining is never + # shortened below the configured value. + relay_state = relay_payload["state"] if relay_payload else None + if relay_state not in (None, "on", "off"): + raise ValueError("relay_entity state must be 'on' or 'off'") + relay_last_changed = relay_payload["last_changed"] if relay_payload else None + if relay_last_changed is not None: + elapsed_seconds = (observed_now - relay_last_changed).total_seconds() + if elapsed_seconds < 0: + entity_blockers.append("relay_timestamp_future") + elapsed_minutes_floor = 0 + else: + elapsed_minutes_floor = int(elapsed_seconds // 60) + else: + elapsed_minutes_floor = None + if relay_entity and relay_payload is None: + entity_blockers.append("relay_entity_unavailable") + initially_on = relay_state == "on" + # The solver wants elapsed OFF time, NOT remaining cooldown. A currently ON + # relay stays initially ON regardless of how recently it was energised. + initial_off_minutes = 0 if initially_on else min(elapsed_minutes_floor, int(off_minutes)) if elapsed_minutes_floor is not None else int(off_minutes) + # Connection entity is informational only. It carries the raw physical + # plug-state string (e.g. '8' plugged, '3' unplugged) with freshness + # uncertain; we surface the observation in the report but never gate + # the relay-driven model on it. + connection_state_observed = connection_payload["state"] if connection_payload else None + connection_in_allowlist = None + if connection_state_observed is not None and connection_entity_states is not None: + connection_in_allowlist = connection_state_observed in connection_entity_states + # Permission entity: exact 'on' means the user has explicitly permitted + # the schedule. Missing observation is reported as not_permitted rather + # than assumed ready. + permission_state = permission_payload["state"] if permission_payload else None + permission_granted = permission_state == "on" + permission_observed = permission_state is not None + return { + "mode": mode, + "settings": deepcopy(settings) if mode == "apply" else None, + "arrival": start_offset, + "control_model": control_model, + "grid_charge_support_w": support_w, + "ev_charging_priority": ev_priority, + "self_consumption_first": self_consumption_first, + "deadline": end_offset, + "departure_passed": departure <= observed_now, + "house_only": mode == "apply" and departure <= observed_now, + "planning_reference_at": now.isoformat(), + "observations_checked_at": observed_now.isoformat(), + "planning_alignment_seconds": (observed_now - now).total_seconds(), + "ev_soc": ev_soc, + "ev_soc_source": ev_soc_source, + "stale_soc": stale_soc, + "allow_stale_shadow_soc": settings.get("allow_stale_shadow_soc", False), + "soc_observed_at": soc_observed_at, + "soc_observation_age_minutes": soc_age_minutes, + "confirmed_observation": confirmed_observation, + "recurring_evidence": {key: recurring[key] for key in ("identity", "session", "latest_deadline")} if recurring else None, + "load_cap": _number(settings.get("load_port_limit_w"), "load_port_limit_w", 1) / 1000, + "import_limit_w": _number(settings.get("import_limit_w", getattr(base, "import_limit", 0)), "import_limit_w", 1), + "pause": pause, + "restart": restart, + "off_minutes": int(off_minutes), + "beam": int(beam), + "terminal_min": _number(settings.get("terminal_min_percent", base.reserve / base.soc_max * 100), "terminal_min_percent", 0, 100) * base.soc_max / 100, + "initially_on": initially_on, + "initial_off_minutes": initial_off_minutes, + "entity_config": { + "relay_entity": relay_entity, + "soc_entity": soc_entity, + "soc_entity_unit": soc_entity_unit, + "connection_entity": connection_entity, + "connection_entity_states": connection_entity_states, + "permission_entity": permission_entity, + }, + "entity_observations": { + "relay_state": relay_state, + "connection_state_observed": connection_state_observed, + "connection_in_allowlist": connection_in_allowlist, + "permission_observed": permission_observed, + "permission_granted": permission_granted, + }, + "elapsed_minutes_since_relay_change": elapsed_minutes_floor, + "entity_blockers": entity_blockers, + } + + +def _collect_blockers(base, options): + """Collect every independent reason the shadow planner cannot run, never first-reason stop.""" + blockers = [] + if base.num_cars != 1 or base.num_inverters != 1: + blockers.append("multi_topology") + if base.iboost_enable: + blockers.append("iboost_unsupported") + if not base.car_energy_reported_load: + blockers.append("load_not_reported") + template_model = options and options["control_model"] == "deye_solarman" + if not base.set_charge_window or (not template_model and not base.set_charge_freeze) or base.set_charge_freeze_only or base.set_charge_low_power: + blockers.append("freeze_required") + if template_model: + from joint_site_template import validate_template + + inverters = getattr(base, "inverters", None) + if not isinstance(inverters, list) or len(inverters) != 1: + blockers.append("single_template_inverter_required") + else: + try: + validate_template(inverters[0]) + except (ValueError, TypeError, KeyError, AttributeError): + blockers.append("template_contract_invalid") + if base.export_limit != 0 or not base.set_discharge_during_charge: + blockers.append("template_dispatch_model_mismatch") + if base.args.get("charge_freeze_service") not in (None, "", []): + # The native target-equals-current branch can call this even without + # an explicit freeze. The narrow template must not acquire that writer. + blockers.append("template_freeze_service_unsupported") + if not base.car_charging_from_battery: + blockers.append("ev_from_battery_required") + if getattr(base, "manual_charge_times", None): + blockers.append("manual_charge_times") + if getattr(base, "manual_demand_times", None): + blockers.append("manual_demand_times") + if getattr(base, "manual_export_times", None): + blockers.append("manual_export_times") + if getattr(base, "manual_freeze_charge_times", None): + blockers.append("manual_freeze_charge_times") + if getattr(base, "manual_freeze_export_times", None): + blockers.append("manual_freeze_export_times") + if getattr(base, "all_active_keep_max", None): + blockers.append("active_keep_ceiling") + if getattr(base, "io_adjusted", None): + blockers.append("io_adjusted") + if type(base.forecast_minutes) is not int or not 0 < base.forecast_minutes <= 2880 or base.forecast_minutes % PREDICT_STEP: + blockers.append("forecast_window") + # Entity evidence blockers from the live observation stage. Manual event + # overrides, missing freeze gates, and stale or absent entity observations + # all share the same first-class blocker treatment. + blockers.extend(options.get("entity_blockers", []) if options else []) + return blockers + + +def _forecast_inputs(base, options): + """Require a complete series; never fill missing load/PV with zero.""" + _number(options["import_limit_w"], "import_limit_w", 1) + _number(base.soc_max, "soc_max", 0.001) + _number(base.soc_kw, "soc_kw", base.reserve, base.soc_max) + _number(base.car_charging_limit[0], "car_charging_limit", 0, base.car_charging_battery_size[0]) + _number(base.car_charging_rate[0], "car_charging_rate", 0.001) + for name in ("inverter_loss", "battery_loss", "battery_loss_discharge", "car_charging_loss"): + _number(getattr(base, name), name, 0.001, 1) + for name in ("inverter_limit", "battery_rate_max_charge", "battery_rate_max_charge_dc", "battery_rate_max_discharge", "export_limit", "battery_rate_max_scaling", "battery_rate_max_scaling_discharge"): + _number(getattr(base, name), name) + if options["pause"] < base.reserve / base.soc_max * 100: + raise ValueError("EV pause threshold must not be below the domestic reserve") + for offset in range(0, base.forecast_minutes, PREDICT_STEP): + for name in ("load_minutes_step", "load_minutes_step10", "pv_forecast_minute_step", "pv_forecast_minute10_step"): + _number(getattr(base, name).get(offset), name) + for minute in range(base.minutes_now + offset, base.minutes_now + offset + PREDICT_STEP): + for name in ("rate_import", "rate_export"): + _number(getattr(base, name).get(minute), name, -1000000, 1000000) + _number(base.all_active_keep.get(minute, 0), "domestic_manual_floor", 0, 100) + + +def build_candidate(base, options): + """Search coupled fixed-rate EV and battery decisions using native forecast inputs.""" + from joint_site import BatteryConfig, EVConfig, PlannerConfig, SiteConfig, Slot, plan + + slots = [] + conservative = options.get("forecast_basis") == "envelope" + for index, minute in enumerate(range(0, base.forecast_minutes, PREDICT_STEP)): + absolute = base.minutes_now + minute + import_rate = sum(base.rate_import[m] for m in range(absolute, absolute + PREDICT_STEP)) / PREDICT_STEP + export_rate = sum(base.rate_export[m] for m in range(absolute, absolute + PREDICT_STEP)) / PREDICT_STEP + pv_step = min(base.pv_forecast_minute_step[minute], base.pv_forecast_minute10_step[minute]) if conservative else base.pv_forecast_minute_step[minute] + load_step = max(base.load_minutes_step[minute], base.load_minutes_step10[minute]) if conservative else base.load_minutes_step[minute] + pv_ac = pv_step * (base.inverter_loss if base.inverter_hybrid else 1) + if not base.inverter_hybrid and base.pv_ac_limit > 0: + pv_ac = min(pv_ac, base.pv_ac_limit * PREDICT_STEP) + floor_percent = max(base.all_active_keep.get(m, 0) for m in range(absolute, absolute + PREDICT_STEP)) + slots.append(Slot(index, minute, PREDICT_STEP, load_step, pv_ac, import_rate, export_rate, floor_percent * base.soc_max / 100)) + battery = BatteryConfig( + capacity_kwh=base.soc_max, + initial_kwh=base.soc_kw, + reserve_kwh=base.reserve, + max_charge_kw=min(base.battery_rate_max_charge, base.inverter_limit) * 60 * base.battery_rate_max_scaling / base.inverter_loss * (min(base.charge_scaling10, 1) if conservative else 1), + max_discharge_kw=min(base.battery_rate_max_discharge, base.inverter_limit) * 60 * base.battery_rate_max_scaling_discharge * base.inverter_loss, + charge_efficiency=base.battery_loss * base.inverter_loss, + discharge_efficiency=base.battery_loss_discharge * base.inverter_loss, + end_min_kwh=max(options["terminal_min"], base.reserve), + cycle_cost_per_kwh=_number(base.metric_battery_cycle, "metric_battery_cycle"), + terminal_value_per_kwh=_number(base.battery_value_rate(base.minutes_now + base.forecast_minutes) * base.metric_battery_value_scaling, "terminal_value"), + ) + ev = EVConfig( + battery_kwh=base.car_charging_battery_size[0], + start_soc_kwh=options["ev_soc"], + stop_soc_kwh=options["ev_soc"] if options.get("house_only") else base.car_charging_limit[0], + rate_kw=base.car_charging_rate[0], + efficiency=base.car_charging_loss, + deadline_min=options["deadline"], + availability=tuple( + s.index + for s in slots + if options["arrival"] <= s.start_min and s.start_min + s.duration_min <= options["deadline"] and (options.get("ev_charging_priority") != "solar" or _solar_surplus_kwh(base, s.start_min) >= base.car_charging_rate[0] * PREDICT_STEP / 60 - 1e-9) + ), + pause_battery_percent=options["pause"], + restart_battery_percent=options["restart"], + min_off_minutes=options["off_minutes"], + initially_on=options["initially_on"], + initial_off_minutes=options["initial_off_minutes"], + ) + support_w = _number(options.get("grid_charge_support_w", 0), "grid_charge_support_w", 0, options["import_limit_w"]) + if support_w and (options.get("mode") not in ("shadow", "apply") or options["control_model"] != "deye_solarman"): + raise ValueError("Grid charge support requires the shared control model") + block = options.get("ev_continuous_block") + if block is not None: + from dataclasses import replace + + if not support_w or len(block) != 2 or any(type(value) is not int or value % PREDICT_STEP for value in block) or not options["arrival"] <= block[0] < block[1] <= options["deadline"]: + raise ValueError("Invalid shadow continuous EV block") + ev = replace(ev, availability=tuple(index for index in ev.availability if block[0] <= slots[index].start_min < block[1])) + site = SiteConfig(options["import_limit_w"] / 1000, base.export_limit * 60, options["load_cap"], support_w / 1000 if support_w else None) + modes = ("demand", "charge") if options["control_model"] == "deye_solarman" else ("demand", "hold", "charge") + result = plan( + slots, + ev, + battery, + site, + PlannerConfig( + beam_width=options["beam"], + allow_limited_discharge=options["control_model"] == "deye_tou", + allowed_battery_modes=modes, + economic_diversity=options.get("mode") == "shadow" and options["control_model"] == "deye_solarman" and options.get("economic_diversity", True), + initial_battery_mode="demand" if options.get("initial_demand_only") else None, + require_ev_start=options.get("ev_charging_priority") == "now" and not options.get("house_only") and options["ev_soc"] < base.car_charging_limit[0] - 0.002, + solar_ev_only=options.get("ev_charging_priority") == "solar", + ), + ) + return result, slots + + +def _validate_candidate_modes(options, result): + """Reject unsupported solver actions before replay/staging, never reinterpret a hold as demand.""" + if options["control_model"] == "deye_solarman": + if any(action.battery_mode not in ("demand", "charge") or getattr(action, "discharge_limit_kw", None) is not None for action in result.actions): + raise UnsupportedJointSite("unsupported_template_candidate_controls") + + +def _native_windows(base, result, slots): + """Translate modes to native charge/freeze windows without altering executable lists.""" + if len(result.actions) != len(slots) or any(action.slot_index != slot.index for action, slot in zip(result.actions, slots)): + raise ValueError("Joint candidate must contain one action for every native forecast step") + ev_windows = [] + charge_windows = [] + limits = [] + for action, slot in zip(result.actions, slots): + start = base.minutes_now + slot.start_min + end = start + slot.duration_min + if action.ev_action == "on": + energy = base.car_charging_rate[0] * slot.duration_min / 60 + ev_windows.append({"start": start, "end": end, "kwh": energy, "average": slot.import_price, "cost": energy * slot.import_price, "octopus": False}) + if action.battery_mode in ("charge", "hold"): + target = base.soc_max if action.battery_mode == "charge" else base.reserve + if charge_windows and charge_windows[-1]["end"] == start and limits[-1] == target: + charge_windows[-1]["end"] = end + else: + charge_windows.append({"start": start, "end": end, "average": slot.import_price}) + limits.append(target) + return ev_windows, charge_windows, limits + + +def _replay(base, options, result, slots, windows, scenario): + """Independently check raw native import, reserve and EV hysteresis using a new Prediction.""" + ev_windows, charge_windows, limits = windows + model = copy(base) + model.car_charging_soc = [options["ev_soc"]] * PREDBAT_MAX_CARS + model.car_charging_soc_next = [None] * PREDBAT_MAX_CARS + full_slot_target = options["ev_soc"] + sum(w["kwh"] for w in ev_windows) * base.car_charging_loss + 1 + model.car_charging_limit_model = [full_slot_target] * PREDBAT_MAX_CARS + model.car_charging_slots = [ev_windows] + [[] for _ in range(PREDBAT_MAX_CARS - 1)] + model.prediction_kernel_enable = False + support_w = _number(options.get("grid_charge_support_w", 0), "grid_charge_support_w", 0, options["import_limit_w"]) + if support_w and (options.get("mode") not in ("shadow", "apply") or options["control_model"] != "deye_solarman"): + raise ValueError("Grid charge support replay requires shared controls") + model.joint_grid_charge_support_w = support_w + model.debug_enable = False + model.metric_standing_charge = 0 + model.import_limit = options["import_limit_w"] + model.joint_preview_charge_limits = {minute: limit * 100 / base.soc_max for window, limit in zip(charge_windows, limits) for minute in range(window["start"], window["end"], PREDICT_STEP)} + limited = any(getattr(action, "discharge_limit_kw", None) is not None for action in result.actions) + if limited: + from joint_site_replay import replay_discharge_limits + + pred = replay_discharge_limits(model, result, slots, scenario) + else: + pred = Prediction(model, base.pv_forecast_minute_step, base.pv_forecast_minute10_step, base.load_minutes_step, base.load_minutes_step10) + pred.run_prediction(limits, charge_windows, [], [], scenario, end_record=base.forecast_minutes, save="test") + diagnostic = getattr(pred, "import_limit_diagnostics", None) + if support_w: + evidence = getattr(pred, "joint_grid_support_import", None) + if not isinstance(evidence, dict) or set(evidence) != set(range(0, base.forecast_minutes, PREDICT_STEP)): + raise ValueError("Incomplete grid support replay evidence") + peaks = [_number(row[1], "grid_support_peak") for row in evidence.values()] + diagnostic = {"checked_minutes": len(peaks) * PREDICT_STEP, "peak_w": max(peaks, default=0), "minutes_over": sum(value - options["import_limit_w"] > 1e-6 for value in peaks) * PREDICT_STEP} + elif not diagnostic or getattr(base, "import_limit", None) != options["import_limit_w"]: + # This PR is independent of the import-diagnostic PR. Native saved grid + # power is rounded to 0.001 kW: use its upper rounding bound, not a false + # exact measurement. When available, the companion raw diagnostic wins. + powers = list(pred.predict_grid_power.values()) + peak = max((max(0, value * 1000 + 0.5) for value in powers), default=0) + diagnostic = {"checked_minutes": len(powers) * PREDICT_STEP, "peak_w": peak, "minutes_over": sum(value * 1000 + 0.5 > options["import_limit_w"] for value in powers) * PREDICT_STEP} + reasons = [] + if not diagnostic or diagnostic["checked_minutes"] != base.forecast_minutes: + reasons.append("incomplete_native_prediction") + elif diagnostic["minutes_over"]: + reasons.append("site_import_exceeded") + trace = pred.predict_soc + rounding = getattr(pred, "rounding_bound_kwh", 0) + reserve_min = min(list(trace.values()) + [pred.final_soc]) + if reserve_min < base.reserve - 0.002: + reasons.append("domestic_reserve") + terminal_rounding = rounding if options["terminal_min"] > base.reserve + 0.002 else 0 + if pred.final_soc - terminal_rounding < max(options["terminal_min"], base.reserve) - 0.002: + reasons.append("terminal_reserve") + previous_on = options["initially_on"] + previous_off_minutes = options["initial_off_minutes"] + off_minutes = previous_off_minutes + for slot_index, (action, slot) in enumerate(zip(result.actions, slots)): + on = action.ev_action == "on" + start_soc = trace[slot.start_min] - (slot_index * 0.00005 if limited else 0) + end_soc = trace.get(slot.start_min + slot.duration_min, pred.final_soc) - ((slot_index + 1) * 0.00005 if limited else 0) + if min(start_soc, end_soc) < getattr(slot, "battery_min_kwh", 0) - 0.002: + reasons.append("domestic_manual_floor") + if on and min(start_soc, end_soc) <= options["pause"] * base.soc_max / 100 + 0.002: + reasons.append("ev_pause_threshold") + if on and not previous_on and start_soc < options["restart"] * base.soc_max / 100 + 0.002: + reasons.append("ev_restart_threshold") + if on and (slot.start_min < options["arrival"] or slot.start_min + slot.duration_min > options["deadline"]): + reasons.append("ev_availability_or_deadline") + if on and not previous_on and off_minutes < options["off_minutes"]: + reasons.append("ev_cooldown") + scenario_load = base.load_minutes_step10 if scenario == 1 else base.load_minutes_step + load_kw = scenario_load[slot.start_min] * 60 / slot.duration_min + (base.car_charging_rate[0] if on else 0) + if load_kw > options["load_cap"] + 1e-9: + reasons.append("load_port_exceeded") + if on and options.get("ev_charging_priority") == "solar": + # PV must cover the full house/EV reservation and the battery cannot + # discharge. Native domestic PV-charging conversion losses remain in + # the raw grid/cost trace; solar-only EV is not a whole-site zero-import claim. + if action.battery_mode != "demand" or end_soc < start_soc - 0.002: + reasons.append("solar_only_native_supply") + if _solar_surplus_kwh(base, slot.start_min) < base.car_charging_rate[0] * slot.duration_min / 60 - 1e-9: + reasons.append("solar_surplus_forecast_insufficient") + off_minutes = 0 if on else (0 if previous_on else off_minutes) + slot.duration_min + previous_on = on + ev_target = options["ev_soc"] if options.get("house_only") else base.car_charging_limit[0] + if pred.final_car_soc[0] < ev_target - 0.002: + reasons.append("ev_target_not_met") + check = { + "passed": not reasons, + "reasons": sorted(set(reasons)), + "checked_minutes": diagnostic["checked_minutes"] if diagnostic else 0, + "peak_import_w": diagnostic["peak_w"] if diagnostic else None, + "final_battery_kwh": pred.final_soc, + "final_ev_kwh": pred.final_car_soc[0], + "validated_ev_target_kwh": ev_target, + "minimum_battery_kwh": reserve_min, + "horizon_energy_cost": pred.final_metric - base.cost_today_sofar, + "model": "native_python_grid_charge_support" if support_w else "native_python_segmented_rate_replay" if limited else "native_python_full_power_ev_envelope", + "soc_rounding_bound_kwh": rounding, + "_charge_peak_kwh": [ + max(_number(pred.final_soc if minute == base.forecast_minutes else trace[minute], "charge_window_soc") for minute in range(window["start"] - base.minutes_now, window["end"] - base.minutes_now + PREDICT_STEP, PREDICT_STEP)) + for window in charge_windows + ], + } + # Presentation consumes this exact independent replay, not the native best + # plan or the solver's simpler battery model. A preview failure must never + # change acceptance or interrupt the control cycle. + try: + from joint_site_preview import build_replay_preview + + check["_preview_rows"] = build_replay_preview(model, pred, result, slots, scenario) + except Exception as exc: + check["preview_unavailable"] = True + _log_failure(base, "preview", exc) + return check + + +def _replay_candidate(base, options, result, slots): + """Reduce shadow and opted-in grid-support ceilings to paired peaks plus one percent. + + These are real candidate limits, not cosmetic row values. Replay both scenarios + again with the reduced limits and accept only their independent native checks. + A ceiling cannot override domestic minima, hardware rates or any execution gate. + """ + windows = _native_windows(base, result, slots) + checks = {name: _replay(base, options, result, slots, windows, scenario) for name, scenario in (("nominal", 0), ("pv10", 1))} + peaks = {name: check.pop("_charge_peak_kwh", None) for name, check in checks.items()} + if (options.get("mode") == "shadow" or options.get("grid_charge_support_w")) and all(check["passed"] for check in checks.values()) and all(isinstance(value, list) and len(value) == len(windows[1]) for value in peaks.values()): + limits = [] + for index, original in enumerate(windows[2]): + peak = max(peaks[name][index] for name in ("nominal", "pv10")) + target = min(100, math.ceil(peak * 100 / base.soc_max + 1)) * base.soc_max / 100 + limits.append(target if original == base.soc_max else original) + windows = (windows[0], windows[1], limits) + checks = {name: _replay(base, options, result, slots, windows, scenario) for name, scenario in (("nominal", 0), ("pv10", 1))} + for check in checks.values(): + check.pop("_charge_peak_kwh", None) + return windows, checks + + +def _best_effort_shadow(base, options): + """Find a new paired-safe partial EV schedule without changing the requested target. + + At most eight bounded searches prefer more EV delivery before cost. A search miss is + not infeasibility: the result is the best validated alternative found, never + a claim of maximum reachability. Apply requires separate explicit partial + authorisation and shared support; closed sessions cannot use it. + """ + attempts = [] + authorised = options.get("mode") == "apply" and base.args["joint_site"].get("ev_partial_target_authorised") is True and options.get("grid_charge_support_w") + if (options.get("mode") != "shadow" and not authorised) or options.get("control_model") != "deye_solarman" or options.get("house_only") or options.get("deadline", 0) <= 0: + return None, attempts + requested = base.car_charging_limit[0] + low, high = options["ev_soc"], requested + if requested <= low + 0.002: + return None, attempts + best = None + for _ in range(8): + # Native SOC values are rounded and EV reservations are full five-minute + # steps. One-percent target probes avoid chasing meaningless fractions; + # the actual delivered energy is still taken from both native replays. + step = base.car_charging_battery_size[0] / 100 + target = step * math.floor((low + high) / (2 * step)) + if target <= low + 0.002: + break + model = copy(base) + model.car_charging_limit = list(base.car_charging_limit) + model.car_charging_limit[0] = target + trial = {**options, "forecast_basis": "envelope"} + result, slots = build_candidate(model, trial) + _validate_candidate_modes(options, result) + passed = False + if result.actions: + windows, checks = _replay_candidate(model, trial, result, slots) + passed = all(check["passed"] and _number(check["final_ev_kwh"], "partial_ev_delivery") >= target - 0.002 for check in checks.values()) + if passed: + delivered = min(check["final_ev_kwh"] for check in checks.values()) + if best is None or delivered > best[5] + 0.002 or (abs(delivered - best[5]) <= 0.002 and result.total_cost < best[0].total_cost): + best = (result, slots, windows, checks, target, delivered) + attempts.append({"forecast_basis": "envelope", "beam_width": trial["beam"], "planning_ev_target_kwh": target, "solver_status": result.status, "native_passed": passed}) + if passed: + low = target + else: + high = target + return best, attempts + + +def _solar_surplus_kwh(base, minute): + """Reserve conservative household demand before converting available PV to AC energy.""" + pv = min(base.pv_forecast_minute_step[minute], base.pv_forecast_minute10_step[minute]) + pv *= base.inverter_loss if base.inverter_hybrid else 1 + if not base.inverter_hybrid and base.pv_ac_limit > 0: + pv = min(pv, base.pv_ac_limit * PREDICT_STEP) + pv = min(pv, base.inverter_limit * PREDICT_STEP) + return pv - max(base.load_minutes_step[minute], base.load_minutes_step10[minute]) + + +def _priority_block_starts(base, options, first, last, duration): + """Rank bounded opportunities, with explicit deadline fallback rather than a silent delay.""" + priority = options.get("ev_charging_priority", "cost") + if options["initially_on"]: + return [(0, "active_continuation")] + if priority == "now": + return [(0, "immediate")] if first == 0 else [] + starts = list(range(first, last + 1, PREDICT_STEP)) + ev_step = base.car_charging_rate[0] * PREDICT_STEP / 60 + if priority == "solar": + return [(start, "solar_surplus_only") for start in starts if all(_solar_surplus_kwh(base, minute) >= ev_step - 1e-9 for minute in range(start, start + duration, PREDICT_STEP))][:12] + if priority == "asap": + solar = [start for start in starts if start <= 30 and all(_solar_surplus_kwh(base, minute) >= ev_step - 1e-9 for minute in range(start, start + min(15, duration), PREDICT_STEP))] + current_rate = base.rate_import[base.minutes_now] + previous = base.minutes_now - 1 + while previous >= 0 and base.rate_import.get(previous) == current_rate: + previous -= 1 + current_discount = base.rate_import.get(previous, current_rate) > current_rate + 1e-9 + cheaper = [start for start in starts if start not in solar and (base.rate_import[base.minutes_now + start] < current_rate - 1e-9 or (current_discount and base.rate_import[base.minutes_now + start] == current_rate))] + preferred = [(start, "near_term_solar") for start in solar] + [(start, "next_cheaper_band") for start in cheaper] + # Failed preferred blocks never erase an earlier feasible deadline path. + preferred = preferred[:12] + tried = {start for start, _ in preferred} + return preferred + [(start, "deadline_or_no_opportunity") for start in starts if start not in tried][:12] + # Include PV opportunity cost when selecting the twelve coupled searches. + # The final ranking still uses the whole-house solver objective and both replays. + ranked = [] + for start in starts: + estimate = 0 + for minute in range(start, start + duration, PREDICT_STEP): + solar = min(ev_step, max(0, _solar_surplus_kwh(base, minute))) + absolute = base.minutes_now + minute + estimate += (ev_step - solar) * base.rate_import[absolute] + solar * base.rate_export[absolute] + ranked.append((estimate, start)) + return [(start, "coupled_cost") for _, start in sorted(ranked)[:12]] + + +def _continuous_grid_support_shadow(base, options): + """Prefer one paired-safe fixed-rate block within the finite departure horizon. + + This is an explicit continuity preference, not a calibrated restart penalty or + global cost optimum. Rank valid blocks by the existing conservative solver + objective (including cycle cost and terminal energy). COST also compares an + unrestricted envelope schedule, so continuity cannot hide a cheaper valid plan. + An active relay tries continuation without inventing a new OFF interval. + ASAP tries nearby solar, then the next cheaper band, then an earlier safe + deadline path. SOLAR reserves conservative surplus and never falls back to grid. + NOW checks only the current interval; every other search also requires the + initial EV action, so no retry or partial alternative can silently defer it. + The search remains bounded; failure is not proof an earlier start is impossible. + Closed sessions cannot use this preference. + """ + if not options.get("grid_charge_support_w") or options.get("mode") not in ("shadow", "apply") or options.get("control_model") != "deye_solarman" or options.get("house_only"): + return None, [] + required = max(base.car_charging_limit[0] - options["ev_soc"], 0) + duration = math.ceil(required / (base.car_charging_rate[0] * base.car_charging_loss) * 60 / PREDICT_STEP - 1e-9) * PREDICT_STEP + first = 0 if options["initially_on"] else max(options["arrival"], math.ceil(max(options["off_minutes"] - options["initial_off_minutes"], 0) / PREDICT_STEP) * PREDICT_STEP) + last = options["deadline"] - duration + if duration <= 0 or last < first: + return None, [] + priority = options.get("ev_charging_priority", "cost") + ranked = _priority_block_starts(base, options, first, last, duration) + best, attempts = None, [] + selected = None + for start, reason in ranked: + for initial_demand in (True, False) if options.get("self_consumption_first") else (False,): + trial = {**options, "forecast_basis": "envelope", "ev_continuous_block": (start, start + duration), "initial_demand_only": initial_demand} + result, slots = build_candidate(base, trial) + passed = False + if result.actions: + _validate_candidate_modes(options, result) + windows, checks = _replay_candidate(base, trial, result, slots) + passed = all(check["passed"] for check in checks.values()) + if passed and (best is None or result.total_cost < best[0].total_cost): + best = result, slots, windows, checks + selected = len(attempts) + attempts.append({"start_offset": start, "end_offset": start + duration, "solver_status": result.status, "native_passed": passed, "initial_demand_only": initial_demand, "priority_reason": reason}) + if passed: + break + if priority != "cost" and passed: + break + if priority == "cost" and not options["initially_on"]: + trial = {**options, "forecast_basis": "envelope"} + result, slots = build_candidate(base, trial) + passed = False + if result.actions: + _validate_candidate_modes(options, result) + windows, checks = _replay_candidate(base, trial, result, slots) + passed = all(check["passed"] for check in checks.values()) + if passed and (best is None or result.total_cost < best[0].total_cost - 1e-9): + best = result, slots, windows, checks + selected = len(attempts) + attempts.append({"solver_status": result.status, "native_passed": passed, "priority_reason": "coupled_cost_flexible_schedule"}) + if selected is not None: + attempts[selected]["selected"] = True + return best, attempts + + +def _empty_report(status): + """Build fresh, deliberately non-executable output; no stale schedule survives a failed run.""" + return { + "status": status, + "generated_at": _observation_clock().isoformat(), + "ready_to_activate": False, + "execution_enabled": False, + "enforced": False, + "actuator_signals_modified": False, + "proposed_ev_windows": [], + "proposed_charge_windows": [], + "proposed_charge_limits": [], + "proposed_discharge_power_limits": [], + "native_checks": {}, + } + + +def _log_failure(base, stage, error): + """Identify programmer/transport failures without echoing private exception values.""" + try: + base.log("Warn: Joint shadow planner failed during {} ({}); executable plan was not modified".format(stage, type(error).__name__)) + except Exception: + pass + + +def _publish_report(base, report, publish): + """A shadow sensor or logger failure must not abort the native control cycle.""" + # The new per-run report owns its preview. Failed/disabled calculations + # cannot inherit an earlier trajectory, and checks retain their original + # validation schema rather than mixing presentation with acceptance. + preview = {} + for scenario, check in report.get("native_checks", {}).items(): + rows = check.pop("_preview_rows", None) + if rows: + preview[scenario] = rows + if set(preview) == {"nominal", "pv10"}: + report["preview"] = preview + base.joint_site_report = report + if publish: + try: + # Keep the full replay in memory for the read-only web page, not + # duplicated into HA's recorder on every calculation. + attributes = {key: value for key, value in report.items() if key != "preview"} + base.dashboard_item(base.prefix + ".joint_site_plan", state=report["status"], attributes=attributes) + except Exception as exc: + report["publication_failed"] = True + _log_failure(base, "publication", exc) + + +def evaluate_joint_site(base, publish=True): + """Evaluate a candidate; only explicit apply may stage it for later native execution.""" + settings = base.args.get("joint_site") + if (isinstance(settings, dict) and settings.get("mode") == "apply") or getattr(base, "joint_site_pending", None) is not None: + # A previous successful candidate cannot survive a failed calculation. + base.joint_site_pending = None + report = _empty_report("invalid_input") + stage = "configuration" + try: + from joint_site_session import household_reason + + reason = household_reason(base) + if reason: + report = _empty_report("household_fallback") + report.update(execution_reason=reason, requested_mode="apply", ev_available=False, household_executor="native", ev_soc_not_required=True) + from joint_site_recurring import recurring_enabled, resolve_recurring_session + + if recurring_enabled(settings): + recurring = resolve_recurring_session(base) + report["recurring_evidence"] = {"status": recurring["status"], "latest_deadline": recurring.get("latest_deadline")} + from joint_site_arrival import arrival_evidence + + report["arrival_forecast"] = arrival_evidence(base) + _publish_report(base, report, publish) + return report + options = _options(base) + if options is None: + if getattr(base, "joint_site_report", None): + report = _empty_report("disabled") + _publish_report(base, report, publish) + return None + stage = "evidence" + model_base = base + execution_requirements = [] + if options["control_model"] == "deye_tou" and options["mode"] == "shadow": + # A counterfactual MODEL, never a capability override on the actual + # inverter. The separate DEYE adapter defines these ToU actions; + # its commissioning and native command equivalence remain gates. + model_base = copy(base) + model_base.set_charge_freeze = True + model_base.set_discharge_during_charge = True + model_base.set_charge_low_power = False + model_base.export_limit = 0 + execution_requirements = ["deye_hardware_commissioning", "standing_hardware_current_limits", "native_executor_equivalence", "connection_and_session_permission", "tou_table_validation"] + evidence = { + "ev_initial_state": { + "initially_on": options["initially_on"], + "initial_off_minutes": options["initial_off_minutes"], + "ev_soc_kwh": options["ev_soc"], + "ev_soc_source": options["ev_soc_source"], + **options["entity_observations"], + }, + "ready_to_activate": False, + "control_model": options["control_model"], + "grid_charge_support_w": options["grid_charge_support_w"], + "ev_charging_priority": options["ev_charging_priority"], + "self_consumption_first": options["self_consumption_first"], + "grid_charge_support_convention": "charge_request_with_load_priority_and_bounded_battery_support" if options["grid_charge_support_w"] else "disabled", + "confirmed_observation": options["confirmed_observation"], + "recurring_evidence": options["recurring_evidence"], + "counterfactual_controls": options["control_model"] == "deye_tou" and options["mode"] == "shadow", + "requested_mode": options["mode"], + "activation_requirements": execution_requirements, + "manual_domestic_floor_slots": len(base.all_active_keep), + "model_export_limit_w": model_base.export_limit * 60000, + "planning_reference_at": options["planning_reference_at"], + "observations_checked_at": options["observations_checked_at"], + "soc_observed_at": options["soc_observed_at"], + "soc_observation_age_minutes": options["soc_observation_age_minutes"], + "planning_alignment_seconds": options["planning_alignment_seconds"], + "session_closed": options["departure_passed"], + "ev_session_renewed": False, + } + if options["departure_passed"]: + # A house-only continuation is not evidence of an EV deadline met. + # Keep the actual measured SOC; never invent a full/zero EV value. + evidence["observed_ev_target_met"] = None if options["ev_soc"] is None or options["ev_soc_source"] == "closed_session_baseline" else options["ev_soc"] >= base.car_charging_limit[0] + evidence["target_met_by_departure"] = None + blockers = _collect_blockers(model_base, options) + stale_hypothesis = options["mode"] == "shadow" and options["allow_stale_shadow_soc"] and options["stale_soc"] is not None and blockers == ["soc_entity_stale_or_future"] and not options["departure_passed"] + if stale_hypothesis: + # Preserve the freshness failure. This is a separately labelled + # mathematical scenario, not a replacement live observation. + options = {**options, "ev_soc": options["stale_soc"], "ev_soc_source": "stale_entity_assumption"} + evidence["uses_stale_soc_assumption"] = True + evidence["input_blockers"] = list(blockers) + evidence["activation_requirements"] = [*evidence["activation_requirements"], "fresh_ev_soc_observation"] + evidence["ev_initial_state"] = {**evidence["ev_initial_state"], "ev_soc_kwh": options["ev_soc"], "ev_soc_source": options["ev_soc_source"]} + # Always surface the entity observations even when topology blockers + # gate the solver. Missing optional entities are reported explicitly + # instead of being silently assumed away. + entity_observations = options.get("entity_observations", {}) + if options["departure_passed"] and not options["house_only"]: + report = _empty_report("departure_passed") + report.update(evidence) + report["observed_ev_target_met"] = None if options["ev_soc"] is None else options["ev_soc"] >= base.car_charging_limit[0] + report["target_met_by_departure"] = None + report["reason"] = "Dated session closed; a later SOC reading cannot prove the departure target was met on time" + _publish_report(base, report, publish) + return report + if blockers and not stale_hypothesis: + report = _empty_report("unsupported") + report["execution_blockers"] = list(blockers) + report["reason"] = "Shadow planner is not configured for the current evidence set" + report.update(evidence) + report["entity_observations"] = entity_observations + _publish_report(base, report, publish) + return report + stage = "forecast" + report.update(evidence) + _forecast_inputs(model_base, options) + stage = "solver" + result, slots = build_candidate(model_base, options) + continuous = None + if options["grid_charge_support_w"]: + continuous, report["continuous_block_attempts"] = _continuous_grid_support_shadow(model_base, options) + if continuous is not None: + result, slots = continuous[:2] + report["ev_run_strategy"] = "single_continuous_block_preferred_no_restart_calibration" if continuous is not None else "ordinary_bounded_search" + if options["ev_charging_priority"] == "asap": + report["ev_run_strategy"] = "next_solar_or_cheaper_opportunity" if continuous is not None else "asap_deadline_fallback_bounded_search" + elif options["ev_charging_priority"] == "now": + report["ev_run_strategy"] = "immediate_paired_continuous_block" if continuous is not None else "immediate_start_required_bounded_search" + elif options["ev_charging_priority"] == "solar": + report["ev_run_strategy"] = "solar_surplus_only" + elif continuous is not None: + report["ev_run_strategy"] = "lowest_validated_coupled_cost_found" + report["priority_selection_reason"] = next((attempt["priority_reason"] for attempt in report["continuous_block_attempts"] if attempt.get("selected")), "bounded_search_fallback") + if options["control_model"] == "deye_solarman": + report["supported_battery_modes"] = ["demand", "charge"] + report["search_beam_attempts"] = [{"beam_width": options["beam"], "solver_status": result.status}] + if not result.actions and options["beam"] < 512: + # A bounded search miss is not infeasibility. One explicit, reported + # retry retains the exact capabilities/inputs and stays within the cap. + retry_options = {**options, "beam": 512} + result, slots = build_candidate(model_base, retry_options) + report["search_beam_attempts"].append({"beam_width": 512, "solver_status": result.status}) + _validate_candidate_modes(options, result) + if not result.actions: + report["status"] = "conditional_no_plan_found" if stale_hypothesis else "no_plan_found" + report["execution_blockers"] = list(blockers) + report["entity_observations"] = entity_observations + report["initial_execution_state_assumption"] = "relay_value_and_elapsed_off_time" if options["entity_config"]["relay_entity"] else "assumed_off_for_minimum_cooldown_no_relay" + else: + stage = "native_replay" + windows, checks = continuous[2:] if continuous is not None else _replay_candidate(model_base, options, result, slots) + selected_basis = "envelope" if continuous is not None else "nominal" + attempts = [{"forecast_basis": selected_basis, "solver_status": result.status, "native_passed": all(check["passed"] for check in checks.values())}] + if options["control_model"] in ("deye_tou", "deye_solarman") and not attempts[0]["native_passed"]: + # One bounded retry uses the forecast envelope and PV10 charge + # delivery assumption. Native nominal AND PV10 still validate + # the same full-rate commands; this is not charge modulation. + retry_options = {**options, "forecast_basis": "envelope"} + retry_result, retry_slots = build_candidate(model_base, retry_options) + _validate_candidate_modes(options, retry_result) + retry_checks = {} + if options["control_model"] == "deye_solarman" and not retry_result.actions and retry_options["beam"] < 512: + attempts.append({"forecast_basis": "envelope", "beam_width": retry_options["beam"], "solver_status": retry_result.status, "native_passed": False}) + retry_options = {**retry_options, "beam": 512} + retry_result, retry_slots = build_candidate(model_base, retry_options) + _validate_candidate_modes(options, retry_result) + if retry_result.actions: + retry_windows, retry_checks = _replay_candidate(model_base, retry_options, retry_result, retry_slots) + result, slots, windows, checks = retry_result, retry_slots, retry_windows, retry_checks + selected_basis = "envelope" + attempts.append({"forecast_basis": "envelope", "beam_width": retry_options["beam"], "solver_status": retry_result.status, "native_passed": bool(retry_checks) and all(check["passed"] for check in retry_checks.values())}) + if options["mode"] == "shadow" and options["control_model"] == "deye_solarman" and not all(check["passed"] for check in checks.values()): + # Different finite pruning orders can lose different valid paths. + # Recover once with the established order; never relax replay gates. + fallback_options = {**options, "forecast_basis": "envelope", "economic_diversity": False} + fallback_result, fallback_slots = build_candidate(model_base, fallback_options) + fallback_checks = {} + if fallback_result.actions: + _validate_candidate_modes(options, fallback_result) + fallback_windows, fallback_checks = _replay_candidate(model_base, fallback_options, fallback_result, fallback_slots) + if all(check["passed"] for check in fallback_checks.values()): + result, slots, windows, checks = fallback_result, fallback_slots, fallback_windows, fallback_checks + selected_basis = "envelope" + attempts.append( + { + "forecast_basis": "envelope", + "beam_width": options["beam"], + "pruning_order": "established", + "solver_status": fallback_result.status, + "native_passed": bool(fallback_checks) and all(check["passed"] for check in fallback_checks.values()), + } + ) + from joint_site_replay import discharge_limit_w + + power_limits = [] + command_steps = [] + for action, slot in zip(result.actions, slots): + power = discharge_limit_w(model_base, action) + command_steps.append( + { + "start": model_base.minutes_now + slot.start_min, + "end": model_base.minutes_now + slot.start_min + slot.duration_min, + "target_soc": next( + (round(limit * 100 / model_base.soc_max) for window, limit in zip(windows[1], windows[2]) if window["start"] <= model_base.minutes_now + slot.start_min < window["end"]), + 0, + ) + if action.battery_mode == "charge" + else 0, + "freeze": action.battery_mode == "hold", + "power_w": power, + "ev_on": action.ev_action == "on", + } + ) + if power is not None: + power_limits.append({"start": model_base.minutes_now + slot.start_min, "end": model_base.minutes_now + slot.start_min + slot.duration_min, "power_w": power}) + command_check = {"available": False, "passed": False, "reason": "Optional local DEYE adapter/mapping is not installed"} + if options["control_model"] == "deye_tou" and base.args.get("deye_shadow"): + try: + from deye_local_shadow import check_joint_deye_commands + + command_check = check_joint_deye_commands(base, command_steps) + except ImportError as exc: + command_check["reason"] = "Local DEYE command-check module is unavailable or incompatible" + command_check["error_type"] = type(exc).__name__ + report.update( + status=("conditional_" if stale_hypothesis else "") + ("candidate" if all(check["passed"] for check in checks.values()) else "native_rejected"), + proposed_ev_windows=windows[0], + proposed_charge_windows=windows[1], + proposed_charge_limits=windows[2], + native_checks=checks, + search_attempts=attempts, + proposed_discharge_power_limits=power_limits, + deye_command_check=command_check, + solver_forecast_basis=selected_basis, + solver_objective=result.total_cost, + solver_grid_cost=sum(action.grid_import_kwh * action.import_price - action.grid_export_kwh * action.export_price for action in result.actions), + cost_unit=base.currency_symbols[1], + cost_convention="solver_objective_includes_cycle_cost_and_terminal_credit; native_horizon_energy_cost_excludes_both_and_standing_charge", + ev_window_convention="full_step_power_reservation_not_partial_target_energy", + charge_limit_convention="paired_native_replay_window_peak_plus_one_percentage_point_revalidated" if options["mode"] == "shadow" or options["grid_charge_support_w"] else "native_full_capacity_ceiling", + initial_execution_state_assumption="relay_value_and_elapsed_off_time" if options["entity_config"]["relay_entity"] else "assumed_off_for_minimum_cooldown_no_relay", + ev_initial_state={ + "initially_on": options["initially_on"], + "initial_off_minutes": options["initial_off_minutes"], + "elapsed_minutes_since_relay_change": options["elapsed_minutes_since_relay_change"], + "ev_soc_source": options["ev_soc_source"], + "ev_soc_kwh": options["ev_soc"], + }, + entity_observations=entity_observations, + solver_ev_ac_kwh=result.diagnostics.ev_delivered_total_kwh, + solver_pv_curtailed_kwh=result.diagnostics.pv_curtailed_total_kwh, + bounded_search=True, + execution_blockers=list(blockers), + ) + if options["grid_charge_support_w"] and all(check["passed"] for check in checks.values()): + delivered = min(check["final_ev_kwh"] for check in checks.values()) + requested = model_base.car_charging_limit[0] + report.update( + requested_ev_target_kwh=requested, + requested_ev_target_percent=requested * 100 / model_base.car_charging_battery_size[0], + ev_departure_at=options["settings"]["session"]["departure"] if options["settings"] else base.args["joint_site"]["session"]["departure"], + planning_ev_target_kwh=requested, + planned_ev_kwh_by_departure=min(delivered, model_base.car_charging_battery_size[0]), + planned_ev_percent_by_departure=min(100, delivered * 100 / model_base.car_charging_battery_size[0]), + requested_ev_target_met=delivered >= requested - 0.002, + ev_target_shortfall_kwh=max(0, requested - delivered), + ) + if options["mode"] == "apply" and report["status"] == "candidate": + from joint_site_apply import execution_fingerprint, command_fingerprint + + report["apply_candidate_staged"] = True + base.joint_site_pending = { + "report": deepcopy(report), + "options": deepcopy(options), + "commands": deepcopy(command_steps), + "ev_windows": deepcopy(windows[0]), + "initial_battery_soc_kwh": base.soc_kw, + "fingerprint": execution_fingerprint(base), + } + if options["grid_charge_support_w"]: + base.joint_site_pending["command_fingerprint"] = command_fingerprint(command_steps) + base.joint_site_pending["replay_inputs"] = deepcopy((result, slots, windows)) + partial_authorised = options["mode"] == "apply" and settings.get("ev_partial_target_authorised") is True and bool(options["grid_charge_support_w"]) + if (options["mode"] == "shadow" or partial_authorised) and report["status"] in ("no_plan_found", "conditional_no_plan_found", "native_rejected", "conditional_native_rejected"): + stage = "partial_shadow_search" + partial, partial_attempts = _best_effort_shadow(model_base, options) + report["partial_search_attempts"] = partial_attempts + if partial is not None: + result, slots, windows, checks, planning_target, delivered = partial + if partial_authorised and options["ev_charging_priority"] in ("asap", "now", "solar"): + partial_base = copy(model_base) + partial_base.car_charging_limit = list(model_base.car_charging_limit) + partial_base.car_charging_limit[0] = planning_target + continuous, report["partial_continuous_block_attempts"] = _continuous_grid_support_shadow(partial_base, options) + if continuous is not None: + result, slots, windows, checks = continuous + delivered = min(check["final_ev_kwh"] for check in checks.values()) + report["ev_run_strategy"] = {"now": "immediate_paired_partial_continuous_block", "asap": "next_opportunity_partial_continuous_block", "solar": "solar_surplus_only_partial_block"}[options["ev_charging_priority"]] + report["priority_selection_reason"] = next(attempt["priority_reason"] for attempt in report["partial_continuous_block_attempts"] if attempt.get("selected")) + requested = model_base.car_charging_limit[0] + target_met = delivered >= requested - 0.002 + report.update( + full_target_search_status=report["status"], + status=("conditional_" if stale_hypothesis else "") + ("candidate" if target_met else "partial_candidate"), + proposed_ev_windows=windows[0], + proposed_charge_windows=windows[1], + proposed_charge_limits=windows[2], + native_checks=checks, + proposed_discharge_power_limits=[], + solver_forecast_basis="envelope", + solver_objective=result.total_cost, + solver_grid_cost=sum(action.grid_import_kwh * action.import_price - action.grid_export_kwh * action.export_price for action in result.actions), + requested_ev_target_kwh=requested, + requested_ev_target_percent=requested * 100 / model_base.car_charging_battery_size[0], + ev_departure_at=options["settings"]["session"]["departure"] if options["settings"] else base.args["joint_site"]["session"]["departure"], + planning_ev_target_kwh=planning_target, + planned_ev_kwh_by_departure=delivered, + planned_ev_percent_by_departure=delivered * 100 / model_base.car_charging_battery_size[0], + requested_ev_target_met=target_met, + ev_target_shortfall_kwh=max(0, requested - delivered), + partial_plan_convention="best_paired_validated_ev_delivery_found_not_maximum_reachability_proof", + solver_ev_ac_kwh=result.diagnostics.ev_delivered_total_kwh, + solver_pv_curtailed_kwh=result.diagnostics.pv_curtailed_total_kwh, + bounded_search=True, + execution_enabled=False, + enforced=False, + actuator_signals_modified=False, + execution_blockers=list(blockers), + ev_initial_state={**evidence["ev_initial_state"], "ev_soc_kwh": options["ev_soc"], "ev_soc_source": options["ev_soc_source"]}, + charge_limit_convention="paired_native_replay_window_peak_plus_one_percentage_point_revalidated", + cost_unit=base.currency_symbols[1], + cost_convention="solver_objective_includes_cycle_cost_and_terminal_credit; native_horizon_energy_cost_excludes_both_and_standing_charge", + ev_window_convention="full_step_power_reservation_not_partial_target_energy", + entity_observations=entity_observations, + ) + if partial_authorised: + from joint_site_session import partial_fingerprint, partial_metadata_valid + from joint_site_apply import execution_fingerprint, command_fingerprint + + report["partial_target_authorised"] = True + if not partial_metadata_valid(base, report): + raise ValueError("Inconsistent partial replay evidence") + commands = [] + for action, slot in zip(result.actions, slots): + target = next((round(limit * 100 / base.soc_max) for window, limit in zip(windows[1], windows[2]) if window["start"] <= base.minutes_now + slot.start_min < window["end"]), 0) + commands.append( + { + "start": base.minutes_now + slot.start_min, + "end": base.minutes_now + slot.start_min + slot.duration_min, + "target_soc": target if action.battery_mode == "charge" else 0, + "freeze": False, + "power_w": None, + "ev_on": action.ev_action == "on", + } + ) + report["apply_candidate_staged"] = True + base.joint_site_pending = { + "report": deepcopy(report), + "options": deepcopy(options), + "commands": commands, + "ev_windows": deepcopy(windows[0]), + "initial_battery_soc_kwh": base.soc_kw, + "fingerprint": execution_fingerprint(base), + "command_fingerprint": command_fingerprint(commands), + "replay_inputs": deepcopy((result, slots, windows)), + "partial_fingerprint": partial_fingerprint(report), + } + if options["ev_charging_priority"] == "solar" and not options.get("house_only"): + report["solar_policy"] = "conservative_household_surplus_only_with_fresh_meter_gate_no_grid_or_battery_ev_fallback" + if report["status"] in ("no_plan_found", "conditional_no_plan_found", "native_rejected", "conditional_native_rejected"): + requested = model_base.car_charging_limit[0] + report.update( + reason="No paired-safe solar-only EV delivery was validated; waiting for surplus without grid fallback", + requested_ev_target_kwh=requested, + requested_ev_target_percent=requested * 100 / base.car_charging_battery_size[0], + planned_ev_kwh_by_departure=options["ev_soc"], + planned_ev_percent_by_departure=options["ev_soc"] * 100 / base.car_charging_battery_size[0], + requested_ev_target_met=options["ev_soc"] >= requested - 0.002, + ev_target_shortfall_kwh=max(0, requested - options["ev_soc"]), + priority_selection_reason="solar_surplus_not_validated", + ) + if options["ev_charging_priority"] == "now" and not options.get("house_only") and options["ev_soc"] < model_base.car_charging_limit[0] - 0.002: + if report["status"] in ("no_plan_found", "conditional_no_plan_found", "native_rejected", "conditional_native_rejected"): + report["ev_run_strategy"] = "immediate_start_not_validated" + report["execution_blockers"] = [*report.get("execution_blockers", []), "immediate_start_not_validated"] + report["reason"] = "No paired-safe EV start was validated for the current control interval; NOW does not defer the start" + except UnsupportedJointSite as exc: + report = _empty_report("unsupported") + report["reason"] = str(exc) + except (ValueError, TypeError, KeyError, AttributeError) as exc: + input_stage = stage in ("configuration", "forecast", "evidence") + report = _empty_report("invalid_input" if input_stage else "error") + report["reason"] = "Check the dated session, supported settings and complete finite native forecasts" if input_stage else "Shadow calculation failed; executable native plan was not modified" + if not input_stage: + report["failure_stage"] = stage + report["error_type"] = type(exc).__name__ + _log_failure(base, stage, exc) + except Exception as exc: + report = _empty_report("error") + report["reason"] = "Unexpected shadow-planner failure; executable native plan was not modified" + report["failure_stage"] = stage + report["error_type"] = type(exc).__name__ + _log_failure(base, stage, exc) + _publish_report(base, report, publish) + return report diff --git a/apps/predbat/joint_site_observation.py b/apps/predbat/joint_site_observation.py new file mode 100644 index 000000000..2c3114dad --- /dev/null +++ b/apps/predbat/joint_site_observation.py @@ -0,0 +1,92 @@ +"""Explicit operator-confirmed EV sessions with measured AC delivery, not fabricated freshness.""" + + +def confirmed_session_observation(base, settings, now): + """Use a dated operator baseline plus the dedicated non-resetting EV energy counter. + + Mercedes push values may remain unchanged while parked. A retained VALID + value is not a fresh measurement, nor is its old last_updated proof of an + invalid value. This opt-in records the operator's confirmation separately, + estimates delivery from the local meter, and compares later valid SOC changes. + It never rewrites entity timestamps, renews departure, or changes targets. + """ + from joint_site_native import _number, _parse_entity_timestamp, _parse_soc_raw, _read_entity_state + + config = settings["confirmed_session"] + if not isinstance(config, dict) or set(config) != {"confirmed_at", "departure", "soc_kwh", "energy_entity", "energy_kwh", "power_entity"}: + raise ValueError("Invalid confirmed EV session") + confirmed = _parse_entity_timestamp(config["confirmed_at"], "operator_confirmation") + departure = _parse_entity_timestamp(config["departure"], "confirmed_departure") + arrival = _parse_entity_timestamp(settings["session"]["available_from"], "session_arrival") + if departure != _parse_entity_timestamp(settings["session"]["departure"], "session_departure") or not arrival <= confirmed <= now or not confirmed < departure or (departure - confirmed).total_seconds() > 86400: + raise ValueError("Invalid or renewed operator session") + capacity = _number(base.car_charging_battery_size[0], "ev_capacity", 0.001) + initial = _number(config["soc_kwh"], "confirmed_soc", 0, capacity) + baseline = _number(config["energy_kwh"], "confirmed_meter") + efficiency = _number(base.car_charging_loss, "ev_efficiency", 0.001, 1) + for name in ("energy_entity", "power_entity"): + if not isinstance(config[name], str) or not config[name].startswith("sensor."): + raise ValueError("Dedicated EV meter required") + if now >= departure: + # This numerical baseline is used only to build a zero-delivery EV model. + # It is not a present SOC reading or evidence of departure success. + return { + "ev_soc": initial, + "ev_soc_source": "closed_session_baseline", + "soc_observed_at": confirmed.isoformat(), + "soc_observation_age_minutes": (now - confirmed).total_seconds() / 60, + "operator_confirmed_at": confirmed.isoformat(), + "ev_metered_ac_kwh": None, + "ev_soc_is_estimate": True, + "session_closed": True, + } + meter = _read_entity_state(base, config["energy_entity"]) + power = _read_entity_state(base, config["power_entity"]) + soc = _read_entity_state(base, settings.get("soc_entity")) + connection = _read_entity_state(base, settings.get("connection_entity")) + relay = _read_entity_state(base, settings.get("relay_entity")) + if any(value is None or not isinstance(value["attributes"], dict) or value["last_updated"] > now or value["last_changed"] > now for value in (meter, power, soc, connection, relay)): + raise ValueError("Confirmed session observation unavailable or future") + if meter["attributes"].get("unit_of_measurement") != "kWh" or power["attributes"].get("unit_of_measurement") != "kW": + raise ValueError("EV meter units do not match") + if any(value["attributes"].get("retrievalstatus") != "VALID" for value in (soc, connection)): + raise ValueError("EV retained state not valid") + measured_soc = _parse_soc_raw(soc["state"], settings.get("soc_entity_unit", "kWh"), capacity) + if measured_soc is None or not 0 <= measured_soc <= capacity: + raise ValueError("Invalid retained EV SOC") + if isinstance(meter["state"], bool) or isinstance(power["state"], bool): + raise ValueError("Invalid EV meter state") + total = _number(float(meter["state"]), "ev_meter", baseline) + watts_kw = _number(float(power["state"]), "ev_power") + if relay["state"] not in ("off", "on"): + raise ValueError("EV relay unavailable") + energy = total - baseline + elapsed_hours = (now - confirmed).total_seconds() / 3600 + rate = _number(base.car_charging_rate[0], "ev_rate", 0.001) + if energy > min(capacity / efficiency + 0.5, rate * elapsed_hours * 1.1 + 0.1) or watts_kw > rate * 1.1: + raise ValueError("EV meter no longer belongs to the confirmed session") + estimated = min(capacity, initial + energy * efficiency) + changed = soc["last_changed"] > confirmed and abs(measured_soc - initial) > 0.001 + # A new HA value is corroboration, not a fabricated physical observation time. + # Only use it during its recent change interval and when consistent with the + # meter. Otherwise the cumulative meter keeps progressing across cycles/restarts. + recent = changed and (now - soc["last_changed"]).total_seconds() <= 180 + tolerance = max(capacity * 0.05, 0.5) + contradiction = changed and (measured_soc < initial - tolerance or measured_soc > estimated + tolerance or (recent and abs(measured_soc - estimated) > tolerance)) + from joint_site_session import retain_session_validity + + retain_session_validity(base, settings, contradiction, soc["last_changed"], now) + completed = changed and measured_soc >= base.car_charging_limit[0] + value = measured_soc if recent or completed else estimated + if value < base.car_charging_limit[0] and relay["state"] == "on" and min((now - relay["last_changed"]).total_seconds(), (now - meter["last_updated"]).total_seconds()) > 600: + raise ValueError("EV energy meter stopped advancing during charge") + return { + "ev_soc": value, + "ev_soc_source": "changed_vehicle_value_after_confirmation" if recent or completed else "operator_confirmed_metered_session", + "soc_observed_at": soc["last_updated"].isoformat(), + "soc_observation_age_minutes": (now - soc["last_updated"]).total_seconds() / 60, + "operator_confirmed_at": confirmed.isoformat(), + "ev_metered_ac_kwh": energy, + "ev_soc_is_estimate": not (recent or completed), + "vehicle_value_kwh": measured_soc, + } diff --git a/apps/predbat/joint_site_preview.py b/apps/predbat/joint_site_preview.py new file mode 100644 index 000000000..9cac9d656 --- /dev/null +++ b/apps/predbat/joint_site_preview.py @@ -0,0 +1,951 @@ +"""Read-only HTML preview of the joint-site native plan and the candidate schedule. + +This module projects a validated joint candidate into the existing web Plan +table without changing the native executable schedule: + +* building an independent, half-hour aligned replay preview from the native + traces (``base.predict_soc``, ``base.predict_grid_power``) and the solver + ``actions`` so the candidate is shown row-by-row, not as a single + averaging table cell; +* rendering a server-side safe ownership banner, not a second table, and + adapting rows for the existing renderer without editable simulation controls; +* when the parent executor publishes a fresh, unexpired, paired-checked + ``status='applied'`` receipt, surfacing the actual corrected plan + (paired rows, "applied" mode, no "Simulation only — not executed" banner) + instead of the simulated candidate. A stale / missing / future / false + authority receipt clears the rows and surfaces a compact + execution-blocked reason without ever falling back to the unrelated + native plan and without claiming success. + +No inputs are mutated; the module never calls a HA service, never writes to +the inverter and never acquires the executable plan. +""" + +from datetime import datetime, timedelta, timezone +from html import escape +import math + +from const import TIME_FORMAT + + +PREVIEW_ROW_LIMIT = 97 +_NO_CANDIDATE_STATUSES = frozenset( + { + "disabled", + "invalid_input", + "unsupported", + "no_plan_found", + "conditional_no_plan_found", + "error", + "departure_passed", + "native_rejected", + } +) +_EXCEEDED_STATES = ("exceeded", "over", "true", "on", "yes", "1") +_OK_STATES = ("ok", "within", "false", "off", "no", "0") +_STALE_AGE_MINUTES = 15 +# Receipt coherence window: matches the executor's RECEIPT_TTL_SECONDS +# default of 300 seconds. Both ends of the interval are strict so a +# future-dated receipt or a now == observed timestamp is rejected. +_RECEIPT_COHERENCE_WINDOW_SECONDS = 300 +# Bounded labels — the only ``execution_reason`` strings the parent executor +# writes that we are willing to echo verbatim. Anything outside this set is +# surfaced as a generic "execution blocked" tag rather than echoed. +_ALLOWED_APPLIED_REASONS = frozenset({"inverter_observed_not_hardware_confirmed"}) +_ALLOWED_BLOCKED_REASONS = frozenset( + { + "apply_commissioning_required", + "local_deye_required", + "local_deye_adapter_missing", + "deye_commissioning_required", + "inverter_capability_or_calibration", + "read_only", + "native_inputs_or_overrides_blocked", + "native_state_not_ready", + "deye_dispatch_model_mismatch", + "manual_ceiling_unsupported", + "entity_observation_unavailable", + "permission_not_granted", + "connection_not_confirmed", + "physical_observation_stale_or_future", + "entity_timestamp_future", + "ev_cooldown", + "relay_state_invalid", + "import_limit_mismatch", + "domestic_manual_floor", + "discharge_power_out_of_range", + "invalid_target", + "invalid_hold_target", + "invalid_command_boolean", + "command_timeline_mismatch", + "command_expired_during_dispatch", + "command_authority_expired", + "no_pending_candidate", + "candidate_not_accepted", + "candidate_stale_or_future", + "configuration_changed", + "configuration_changed_during_dispatch", + "incomplete_command_horizon", + "inverter_command_not_observed", + "native_checks_failed", + "execution_validation_failed", + "publication_failed", + } +) + + +def _is_finite(value): + """Truthy only for a real, finite number; reject booleans, NaN and inf.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + return math.isfinite(value) + + +def _format_iso(base, absolute_minute): + """Use the same offset-aware absolute clock as the native prediction.""" + midnight = getattr(base, "midnight_utc", None) + if midnight is None: + return "" + return (midnight + timedelta(minutes=absolute_minute)).strftime(TIME_FORMAT) + + +def _parse_iso(value): + """Best-effort parse of an offset-aware ISO timestamp; never raise.""" + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except (TypeError, ValueError): + return None + return parsed if parsed.tzinfo is not None else None + + +def _safe_iso(value): + """Return an unescaped ISO timestamp only when it parses with timezone.""" + return escape(value) if _parse_iso(value) is not None else "" + + +def _age_minutes(timestamp, now): + """Return the age in minutes, or None when the timestamp is missing/invalid.""" + parsed = _parse_iso(timestamp) + if parsed is None or not isinstance(now, datetime) or now.tzinfo is None: + return None + try: + if now.tzinfo is None: + now = now.replace(tzinfo=timezone.utc) + return (now - parsed).total_seconds() / 60.0 + except (TypeError, AttributeError): + return None + + +def _trace_complete(pred, base, start_relative, duration_min): + """Reject any replay that lacks a full native horizon trace; never fill zeros.""" + if pred is None: + return False + grid = getattr(pred, "predict_grid_power", None) + soc = getattr(pred, "predict_soc", None) + if not isinstance(grid, dict) or not isinstance(soc, dict): + return False + for minute in range(start_relative, start_relative + duration_min, 5): + if not _is_finite(grid.get(_format_iso(base, base.minutes_now + minute))): + return False + if not _is_finite(soc.get(minute)): + return False + endpoint = start_relative + duration_min + value = getattr(pred, "final_soc", None) if endpoint == base.forecast_minutes else soc.get(endpoint) + if not _is_finite(value): + return False + return True + + +def _bin_modes(actions_by_start, slice_starts): + """Collect the actual, ordered unique battery modes from the bin's actions.""" + seen = [] + for slice_start in slice_starts: + mode = getattr(actions_by_start.get(slice_start), "battery_mode", None) + if isinstance(mode, str) and mode and mode not in seen: + seen.append(mode) + return ",".join(seen) + + +def build_replay_preview(base, pred, result, slots, scenario): + """Build half-hour aligned replay rows for the joint-site native plan. + + Returns a list of at most ``PREVIEW_ROW_LIMIT`` dictionaries, one per + half-hour bin that overlaps the planning horizon. Each row uses the + absolute ``base.minutes_now + slot.start_min`` clock, never a relative + index, and reports truthful first/last partial-bin start/end ISO + timestamps derived from ``base.midnight_utc``. Rejects incomplete or + non-finite traces rather than zero-filling them. Does not mutate any + input. + """ + if not isinstance(slots, (list, tuple)) or not slots: + return [] + if result is None or getattr(result, "status", None) != "feasible": + return [] + actions = tuple(getattr(result, "actions", ()) or ()) + if len(actions) != len(slots): + return [] + midnight = getattr(base, "midnight_utc", None) + soc_max = getattr(base, "soc_max", None) + if midnight is None or not _is_finite(soc_max) or soc_max <= 0: + return [] + final_soc = getattr(pred, "final_soc", None) + rates = getattr(base, "car_charging_rate", []) + rate_kw = rates[0] if rates else None + if not _is_finite(final_soc) or not _is_finite(rate_kw) or rate_kw <= 0: + return [] + minutes_now = getattr(base, "minutes_now", None) + horizon = getattr(base, "forecast_minutes", None) + if not isinstance(minutes_now, int) or minutes_now % 5 or not isinstance(horizon, int) or horizon <= 0 or horizon % 5: + return [] + if len(slots) * 5 != horizon or getattr(midnight, "tzinfo", None) is None: + return [] + if not _trace_complete(pred, base, 0, horizon): + return [] + load_map = getattr(base, "load_minutes_step10" if scenario == 1 else "load_minutes_step", {}) + pv_map = getattr(base, "pv_forecast_minute10_step" if scenario == 1 else "pv_forecast_minute_step", {}) + bins = {} + for position, (action, slot) in enumerate(zip(actions, slots)): + minute = position * 5 + if slot.start_min != minute or slot.duration_min != 5 or action.slot_index != slot.index: + return [] + if action.ev_action not in ("on", "off") or action.battery_mode not in ("demand", "hold", "charge"): + return [] + load = load_map.get(minute) + pv = pv_map.get(minute) + if not _is_finite(load) or not _is_finite(pv) or load < 0 or pv < 0: + return [] + absolute = minutes_now + minute + start_soc = pred.predict_soc[minute] * 100 / soc_max + end_soc = (final_soc if minute + 5 == horizon else pred.predict_soc[minute + 5]) * 100 / soc_max + peak = max(0.0, pred.predict_grid_power[_format_iso(base, absolute)] * 1000 + 0.5) + row = bins.setdefault( + absolute // 30, + { + "start": _format_iso(base, absolute), + "end": "", + "minutes": 0, + "battery_modes": [], + "ev_on_minutes": 0, + "ev_reserved_kwh": 0.0, + "house_kwh": 0.0, + "pv_kwh": 0.0, + "battery_start_percent": start_soc, + "battery_end_percent": end_soc, + "battery_min_percent": min(start_soc, end_soc), + "grid_peak_w": 0.0, + }, + ) + row["end"] = _format_iso(base, absolute + 5) + row["minutes"] += 5 + if action.battery_mode not in row["battery_modes"]: + row["battery_modes"].append(action.battery_mode) + if action.ev_action == "on": + row["ev_on_minutes"] += 5 + row["ev_reserved_kwh"] += rate_kw * 5 / 60 + if action.battery_mode == "charge": + target = getattr(base, "joint_preview_charge_limits", {}).get(absolute, 100) + if not _is_finite(target) or not 0 < target <= 100.1: + return [] + row["charge_target_percent"] = max(row.get("charge_target_percent", 0), target) + # Native forecast maps contain kWh per step, not average kW. + row["house_kwh"] += load + row["pv_kwh"] += pv + row["battery_end_percent"] = end_soc + row["battery_min_percent"] = min(row["battery_min_percent"], start_soc, end_soc) + row["grid_peak_w"] = max(row["grid_peak_w"], peak) + # Prices and cumulative cost must belong to this replay too. Never + # borrow the native plan's costs when displaying a different schedule. + if _is_finite(getattr(slot, "import_price", None)) and _is_finite(getattr(slot, "export_price", None)): + row["import_rate"] = row.get("import_rate", 0) + slot.import_price * 5 + row["export_rate"] = row.get("export_rate", 0) + slot.export_price * 5 + metrics = getattr(pred, "metric_time", {}) + cost_start = metrics.get(_format_iso(base, absolute)) + cost_end = getattr(pred, "final_metric", None) if minute + 5 == horizon else metrics.get(_format_iso(base, absolute + 5)) + if "total_cost" not in row and _is_finite(cost_start): + row["total_cost"] = cost_start / 100 + if _is_finite(cost_end) and _is_finite(row.get("total_cost")): + row["cost_change"] = cost_end / 100 - row["total_cost"] + durations = row.setdefault("mode_minutes", {}) + durations[action.battery_mode] = durations.get(action.battery_mode, 0) + 5 + if len(bins) > PREVIEW_ROW_LIMIT: + return [] + rows = list(bins.values()) + for row in rows: + row["battery_modes"] = ",".join(row["battery_modes"]) + for key in ("import_rate", "export_rate"): + if key in row: + row[key] /= row["minutes"] + return rows + + +def _applied_receipt_fresh(report, now): + """Accept only fresh, coherent applied execution receipts. + + The parent ``joint_site_apply._report`` populates the report with: + + * ``status='applied'``, + * ``requested_mode='apply'``, + * ``execution_enabled=True``, + * ``actuator_signals_modified=True``, + * ``execution_observed_at`` (ISO) — the instant the executor observed + the inverter accept the command, + * ``execution_valid_until`` (ISO) — the receipt expiry. + + Coherence requires ``now >= observed_at``, ``now < valid_until`` and an + interval ``(valid_until - observed_at) <= 300s``. Anything else — a + missing field, a future stamp, a TTL that has run out, a stale receipt, + ``execution_enabled`` not exactly ``True``, + ``actuator_signals_modified`` not exactly ``True``, + ``requested_mode`` not exactly ``'apply'``, a non-applied status — is + treated as a non-authoritative receipt and yields ``False``. The view + never falls back to the unrelated native plan and never claims success. + """ + if not isinstance(report, dict): + return False + if report.get("status") != "applied": + return False + if report.get("requested_mode") != "apply": + return False + if report.get("execution_enabled") is not True: + return False + if report.get("actuator_signals_modified") is not True: + return False + if "execution_blockers" in report: + blockers = report.get("execution_blockers") + # Fail closed: only an empty list (or absent key) is acceptable. + # Strings, dicts, True/1, or non-empty lists all reject. + if not isinstance(blockers, list) or blockers: + return False + observed = _parse_iso(report.get("execution_observed_at")) + valid_until = _parse_iso(report.get("execution_valid_until")) + if observed is None or valid_until is None: + return False + if not isinstance(now, datetime) or now.tzinfo is None: + return False + interval_seconds = (valid_until - observed).total_seconds() + if interval_seconds <= 0 or interval_seconds > _RECEIPT_COHERENCE_WINDOW_SECONDS: + return False + if now < observed or now >= valid_until: + return False + return True + + +def _applied_display_allowed(report, now, paired): + """The full gate for rendering the applied-plan banner. + + A "valid applied display" requires every one of: + * ``paired`` is not ``None`` (preview rows must be available); + * the receipt authority is fresh (see :func:`_applied_receipt_fresh`); + * the report is *not* a stale-SoC hypothesis; + * the report is *not* marked ``counterfactual_controls``. + + Anything else collapses to the execution-blocked branch — never the + read-only simulation branch and never a fall-through to the unrelated + native plan. + """ + if paired is None: + return False + if not isinstance(report, dict): + return False + if not _applied_receipt_fresh(report, now): + return False + if report.get("uses_stale_soc_assumption") is True: + return False + if report.get("counterfactual_controls") is True: + return False + # Coherence between the report's ``generated_at`` and the receipt's + # observed_at window. A future-dated or way-stale ``generated_at`` is + # not a valid applied display even with a fresh receipt. + generated_at = _parse_iso(report.get("generated_at")) + observed = _parse_iso(report.get("execution_observed_at")) + if generated_at is None or observed is None: + return False + if generated_at > observed or generated_at > now or now - generated_at >= timedelta(minutes=10): + return False + return True + + +def _paired_preview(report, now): + """Accept only complete, fresh and passed paired native trajectories.""" + if not isinstance(report, dict) or report.get("status") not in ("candidate", "conditional_candidate", "partial_candidate", "conditional_partial_candidate", "applied"): + return None + age = _age_minutes(report.get("generated_at"), now) + if age is None or age < -1 or age > _STALE_AGE_MINUTES: + return None + checks = report.get("native_checks", {}) + if not all(isinstance(checks.get(key), dict) and checks[key].get("passed") is True for key in ("nominal", "pv10")): + return None + if report["status"] in ("partial_candidate", "conditional_partial_candidate"): + required_target = ("requested_ev_target_kwh", "requested_ev_target_percent", "planning_ev_target_kwh", "planned_ev_kwh_by_departure", "planned_ev_percent_by_departure", "ev_target_shortfall_kwh") + if not all(_is_finite(report.get(key)) for key in required_target): + return None + partial_mode = report.get("requested_mode") == "shadow" or (report.get("requested_mode") == "apply" and report.get("partial_target_authorised") is True) + if not partial_mode or report.get("requested_ev_target_met") is not False or any(report.get(key) is not False for key in ("execution_enabled", "enforced", "actuator_signals_modified")): + return None + departure = _parse_iso(report.get("ev_departure_at")) + if departure is None or departure <= now or not 0 <= report["planning_ev_target_kwh"] <= report["planned_ev_kwh_by_departure"] < report["requested_ev_target_kwh"]: + return None + if not 0 <= report["planned_ev_percent_by_departure"] < report["requested_ev_target_percent"] <= 100: + return None + if abs(report["planned_ev_percent_by_departure"] - report["planned_ev_kwh_by_departure"] * report["requested_ev_target_percent"] / report["requested_ev_target_kwh"]) > 0.01: + return None + if not all(_is_finite(checks[key].get("final_ev_kwh")) and checks[key]["final_ev_kwh"] >= report["planning_ev_target_kwh"] - 0.002 for key in ("nominal", "pv10")): + return None + if abs(min(checks[key]["final_ev_kwh"] for key in ("nominal", "pv10")) - report["planned_ev_kwh_by_departure"]) > 0.002: + return None + if abs(report["requested_ev_target_kwh"] - report["planned_ev_kwh_by_departure"] - report["ev_target_shortfall_kwh"]) > 0.002: + return None + preview = report.get("preview", {}) + if not isinstance(preview, dict): + return None + nominal, pv10 = preview.get("nominal"), preview.get("pv10") + if not isinstance(nominal, list) or not isinstance(pv10, list) or not nominal or len(nominal) != len(pv10) or len(nominal) > PREVIEW_ROW_LIMIT: + return None + previous = None + required = ("battery_start_percent", "battery_end_percent", "battery_min_percent", "grid_peak_w", "house_kwh", "pv_kwh", "ev_on_minutes", "ev_reserved_kwh", "minutes") + for left, right in zip(nominal, pv10): + for row in (left, right): + if not isinstance(row, dict) or not all(_is_finite(row.get(key)) for key in required): + return None + target = row.get("charge_target_percent", 100) + if not _is_finite(target) or not 0 < target <= 100.1: + return None + start, end = _parse_iso(row.get("start")), _parse_iso(row.get("end")) + if start is None or end is None or not 0 < row["minutes"] <= 30 or (end - start).total_seconds() != row["minutes"] * 60: + return None + # Native SOC is rounded to 0.001 kWh, capacity is not. Retain its + # genuine value (e.g. 100.001%) rather than rejecting a full battery. + if not all(-0.1 <= row[key] <= 100.1 for key in ("battery_start_percent", "battery_end_percent", "battery_min_percent")): + return None + if any(row[key] < 0 for key in ("grid_peak_w", "house_kwh", "pv_kwh", "ev_reserved_kwh")) or not 0 <= row["ev_on_minutes"] <= row["minutes"]: + return None + if row.get("battery_modes") not in ( + "demand", + "charge", + "hold", + "charge,demand", + "demand,charge", + "hold,demand", + "demand,hold", + "charge,hold", + "hold,charge", + "demand,charge,hold", + "charge,demand,hold", + "hold,charge,demand", + "hold,demand,charge", + "demand,hold,charge", + "charge,hold,demand", + ): + return None + if (left["start"], left["end"]) != (right["start"], right["end"]) or (previous is not None and previous != _parse_iso(left["start"])): + return None + if abs(left.get("charge_target_percent", 100) - right.get("charge_target_percent", 100)) > 0.0001: + return None + previous = _parse_iso(left["end"]) + total_minutes = sum(row["minutes"] for row in nominal) + if any(_is_finite(checks[key].get("checked_minutes")) and checks[key]["checked_minutes"] != total_minutes for key in ("nominal", "pv10")): + return None + return nominal, pv10 + + +def build_joint_plan_view(native_plan, report, now=None): + """Project the candidate into the existing web table schema, never executable state. + + No native row, cost, target or rowspan is inherited. Only display metadata + survives. Invalid/expired active reports clear the candidate instead of + quietly presenting the unrelated native plan as a joint trajectory. + + For a valid applied display (``status == 'applied'`` AND paired rows AND + fresh receipt AND no stale-SoC/counterfactual hypothesis AND coherent + ``generated_at``) the same paired preview rows are emitted, but + ``view['joint_execution']`` is ``True`` and ``view['joint_simulation']`` + is ``False`` — this is the actual applied plan, not a simulation. When + the applied display is invalid (expired / missing / future / false + authority / blocked / missing preview / stale ``generated_at``) the rows + are cleared and ``view['execution_blocked']`` carries a bounded + reason. Parent ``status='apply_blocked'`` reports also land here and + preserve the shadow candidate behaviour for shadow statuses. + """ + if not isinstance(report, dict) or report.get("status") in ("disabled", "household_fallback"): + return native_plan + now = now if now is not None else datetime.now(timezone.utc) + native = native_plan if isinstance(native_plan, dict) else {} + view = {key: native[key] for key in ("time", "currency_symbols", "reason_templates", "import_cost_threshold", "export_cost_threshold") if key in native} + view.update({"timestamp": report.get("generated_at"), "num_cars": 1, "iboost_enable": False, "carbon_enable": False, "rows": []}) + view.setdefault("currency_symbols", ["", ""]) + status = report.get("status") + is_applied = status == "applied" + is_blocked = status == "apply_blocked" + paired = _paired_preview(report, now) if status not in (None, "disabled") else None + applied_fresh = _applied_display_allowed(report, now, paired) + if applied_fresh: + view["joint_simulation"] = False + view["joint_execution"] = True + view["execution_observed_at"] = report.get("execution_observed_at") + view["execution_valid_until"] = report.get("execution_valid_until") + elif is_applied or is_blocked: + # Applied or apply_blocked: surface the execution-blocked view with a + # bounded reason. Never fall back to the native plan, never claim + # success, never show the simulated candidate as a substitute. + view["joint_simulation"] = False + view["joint_execution"] = False + view["execution_blocked"] = _applied_block_reason(report, now) + return view + else: + # Shadow / candidate / partial / etc. — keep ``joint_simulation=True`` + # so the renderer treats it as the read-only simulated plan. + view["joint_simulation"] = True + if paired is None: + # Non-applied status with no paired rows (e.g. shadow unavailable, + # off, error). Preserve the original generic-replay behaviour. + view["joint_unavailable"] = True + return view + for row, pv10 in zip(*paired): + start = _parse_iso(row["start"]) + midnight = _parse_iso(view.get("time")) or start.replace(hour=0, minute=0, second=0, microsecond=0) + names = row["battery_modes"].split(",") + # Keep the established table's state labels, arrows and colour rules. + # Coarse mixed rows use its existing asterisk/reason convention. + change = row["battery_end_percent"] - row["battery_start_percent"] + soc_sym = "→" if abs(change) < 0.15 else ("↗" if change > 0 else "↘") + state = "Chrg" if "charge" in names else ("HoldChrg" if "hold" in names else "Demand") + state_text = "Chrg↗" if state == "Chrg" else ("HoldChrg→" if state == "HoldChrg" else soc_sym) + if len(names) > 1: + state_text += "*" + reasons = [{"code": "mixed_slot_states", "params": {"states": ", ".join(name.capitalize() for name in names)}}] if len(names) > 1 else [] + rate = row.get("import_rate") + threshold = native.get("import_cost_threshold") + rate_color = "#FFFFFF" + if _is_finite(rate) and _is_finite(threshold): + rate_color = "#74C1FF" if rate <= 0 else ("#3AEE85" if rate <= threshold else ("#F18261" if rate > threshold * 1.5 else "#FFFFAA")) + export_rate = row.get("export_rate") + export_threshold = native.get("export_cost_threshold") + export_color = "#FFFFFF" + if _is_finite(export_rate) and _is_finite(export_threshold): + export_color = "#F18261" if export_rate >= 1.5 * export_threshold else ("#FFFFAA" if export_rate >= export_threshold else ("#48deff" if export_rate > 0.5 * export_threshold else "#dcdcdc")) + cost = row.get("cost_change") + cost_color = "#FFFFFF" + if _is_finite(cost): + cost_color = "#F18261" if cost >= 0.1 else ("#FFFF00" if cost >= 0.005 else ("#3AEE85" if cost <= -0.005 else "#FFFFFF")) + view["rows"].append( + { + "time": row["start"], + "slot_minute": int((start - midnight).total_seconds() / 60), + "import_rate": row.get("import_rate") if _is_finite(row.get("import_rate")) else 0, + "export_rate": row.get("export_rate") if _is_finite(row.get("export_rate")) else 0, + "joint_rates_unavailable": not (_is_finite(row.get("import_rate")) and _is_finite(row.get("export_rate"))), + "rate_color_import": rate_color, + "rate_color_export": export_color, + "cost_color": cost_color, + "state": state, + "state_text": state_text, + "state_color": "#3AEE85" if state == "Chrg" else ("#34DBEB" if state == "HoldChrg" else "#FFFFFF"), + "show_limit": str(round(row.get("charge_target_percent", 100))) if "charge" in names else "", + "reasons": reasons, + "split": False, + "pv_forecast": round(row["pv_kwh"], 2), + "pv_forecast10": round(pv10["pv_kwh"], 2), + "load_forecast": round(row["house_kwh"], 2), + "load_forecast10": round(pv10["house_kwh"], 2), + "clipped": 0, + "car_charging": round(row["ev_reserved_kwh"], 3), + "joint_ev_on_minutes": row["ev_on_minutes"], + "soc_percent": row["battery_start_percent"], + "soc_change": row["battery_end_percent"] - row["battery_start_percent"], + "soc_sym": soc_sym, + "soc_color": "#F18261" if row["battery_start_percent"] < 20 else ("#FFFF00" if row["battery_start_percent"] < 50 else "#3AEE85"), + "load_color": "#F18261" if row["house_kwh"] >= 0.5 else ("#FFFF00" if row["house_kwh"] >= 0.25 else ("#AAFFAA" if row["house_kwh"] > 0 else "#FFFFFF")), + "pv_color": "#FFAAAA" if row["pv_kwh"] >= 0.2 else ("#FFFF00" if row["pv_kwh"] >= 0.1 else "#FFFFFF"), + "car_color": "#FFFF00" if row["ev_reserved_kwh"] > 0 else "#FFFFFF", + "joint_soc_end_percent": row["battery_end_percent"], + "joint_soc_pv10_start_percent": pv10["battery_start_percent"], + "joint_soc_pv10_end_percent": pv10["battery_end_percent"], + "joint_grid_peak_w": row["grid_peak_w"], + "joint_grid_peak_pv10_w": pv10["grid_peak_w"], + "joint_interval_end": row["end"], + "cost_change": row.get("cost_change"), + "total_cost": row.get("total_cost"), + } + ) + last = paired[0][-1] + if _is_finite(last.get("total_cost")) and _is_finite(last.get("cost_change")): + view["totals"] = { + "pv_forecast": round(sum(row["pv_kwh"] for row in paired[0]), 2), + "load_forecast": round(sum(row["house_kwh"] for row in paired[0]), 2), + "car_charging": round(sum(row["ev_reserved_kwh"] for row in paired[0]), 2), + "soc_percent": round(last["battery_end_percent"]), + "total_cost": round(last["total_cost"] + last["cost_change"], 2), + "clipped": 0, + } + return view + + +def _format_finite(value, decimals=2): + """Escape a finite numeric value; never render NaN, inf, bool or strings.""" + if not _is_finite(value): + return "" + return escape("{{:.{0}f}}".format(int(decimals)).format(float(value))) + + +def _parse_native_diag(native_diagnostic): + """Return (state, peak_w, minutes_over, limit_w); None for missing/non-finite.""" + if not isinstance(native_diagnostic, dict): + return None, None, None, None + attributes = native_diagnostic.get("attributes") + if not isinstance(attributes, dict): + attributes = {} + + def _opt(key): + """Accept only finite diagnostic quantities, never malformed state data.""" + value = attributes.get(key) + return value if _is_finite(value) else None + + return native_diagnostic.get("state"), _opt("peak_w"), _opt("minutes_over"), _opt("limit_w") + + +def _is_exceeded(state, peak_w, minutes_over, limit_w): + """True when the native diagnostic says the import limit was exceeded.""" + if isinstance(state, str): + normalised = state.strip().lower() + if normalised in _EXCEEDED_STATES: + return True + if normalised in _OK_STATES: + return False + if _is_finite(peak_w) and _is_finite(limit_w) and peak_w > limit_w: + return True + return False + + +def _applied_block_reason(report, now=None): + """Return a bounded execution-blocked reason. + + Reasons are computed from the *actual* coherence checks (missing + fields, wrong types, interval/valid_until problems, future/expired vs + ``now``). The report's ``execution_reason`` is only echoed when its + value is in a small allowlist of known parent labels; arbitrary + strings are rejected and replaced with a generic tag. Never claims + success; never echoes untrusted text. + """ + if not isinstance(report, dict): + return "no execution receipt" + now = now if isinstance(now, datetime) else None + reason = report.get("execution_reason") + if isinstance(reason, str) and reason in _ALLOWED_BLOCKED_REASONS: + return reason + if isinstance(reason, str) and reason in _ALLOWED_APPLIED_REASONS: + # "inverter_observed_not_hardware_confirmed" is the parent's + # observation-only label. Surface it generically so the user sees + # the practical consequence (still waiting on hardware + # confirmation) without echoing the parent's internal wording. + return "awaiting hardware confirmation" + blockers = report.get("execution_blockers") + if "execution_blockers" in report and not (isinstance(blockers, list) and not blockers): + return "execution blocked: declared blockers" + observed = _parse_iso(report.get("execution_observed_at")) + valid_until = _parse_iso(report.get("execution_valid_until")) + if observed is None and valid_until is None: + return "missing execution_observed_at and execution_valid_until" + if observed is None: + return "missing execution_observed_at" + if valid_until is None: + return "missing execution_valid_until" + if valid_until <= observed: + return "execution_valid_until not after execution_observed_at" + interval = (valid_until - observed).total_seconds() + if interval > _RECEIPT_COHERENCE_WINDOW_SECONDS: + return "execution_valid_until exceeds {}s coherence window".format(_RECEIPT_COHERENCE_WINDOW_SECONDS) + if now is not None and now.tzinfo is not None: + if now < observed: + return "execution_observed_at is in the future" + if now >= valid_until: + return "execution receipt expired" + return "execution receipt not coherent" + + +def _unavailable_reason(report): + """Explain rejection without exposing arbitrary diagnostic strings or stale rows.""" + labels = { + "site_import_exceeded": "grid import limit", + "ev_pause_threshold": "EV battery pause threshold", + "ev_restart_threshold": "EV battery restart threshold", + "domestic_reserve": "house battery reserve", + "terminal_reserve": "end-of-plan battery reserve", + "domestic_manual_floor": "manual house battery minimum", + "ev_target_not_met": "EV departure target", + "ev_cooldown": "EV restart delay", + "load_port_exceeded": "house load-port limit", + } + rejected = [] + checks = report.get("native_checks", {}) + checks = checks if isinstance(checks, dict) else {} + for scenario in ("nominal", "pv10"): + check = checks.get(scenario, {}) + reasons = check.get("reasons", []) if isinstance(check, dict) else [] + reasons = reasons if isinstance(reasons, (list, tuple)) else [] + problems = [label for code, label in labels.items() if code in reasons] + if problems: + rejected.append(("conservative forecast" if scenario == "pv10" else "nominal forecast") + ": " + ", ".join(problems)) + if rejected: + return "Latest candidate failed safety checks (" + "; ".join(rejected) + "). No validated replacement was found; the previous simulation is hidden." + if report.get("status") in ("no_plan_found", "conditional_no_plan_found"): + return "The bounded search did not find a complete validated schedule. This does not prove the departure target is impossible; the previous simulation is hidden." + return "No complete paired native-replay preview is available. Rejected or unavailable candidates are not shown; no earlier schedule is reused." + + +def _household_fallback_debug(report): + """Build the hidden Show-Debug markup for a household-fallback report. + + The branch must never project authority: the existing Plan table, columns, + colours and precision stay unchanged. Only the debug span is augmented, and + the arrival forecast (set by ``evaluate_joint_site`` when the recurring + policy withdrew and the native executor handed back to household control) + is rendered with full HTML escaping so Show Debug surfaces the + low-confidence assumption, empirical range, sample count, source + timestamps and shortfall that the live planning model just used. + + The function is intentionally small: it does not touch ``report``, + ``native_diagnostic`` or any other shared state, and it never calls a + HA service or writes to an inverter. + """ + parts = ['") + return "".join(parts) + parts.append("
") + parts.append("Arrival forecast: ") + parts.append(escape(str(arrival.get("status", "unavailable")))) + status = arrival.get("status") + if status == "anticipated": + parts.append("
") + parts.append("Expected arrival: ") + parts.append(_safe_iso(arrival.get("expected_arrival"))) + parts.append(" (p10 ") + parts.append(_safe_iso(arrival.get("arrival_p10"))) + parts.append(", p90 ") + parts.append(_safe_iso(arrival.get("arrival_p90"))) + parts.append(")") + parts.append("
") + parts.append("Confidence: ") + parts.append(escape(str(arrival.get("confidence", "low")))) + parts.append("
") + parts.append("Sample size: ") + parts.append(escape("{} eligible of {} completed trips ({})".format(arrival.get("eligible_trips", 0), arrival.get("completed_trips", 0), arrival.get("cohort", "")))) + parts.append("
") + parts.append("Source timestamp: ") + parts.append(_safe_iso(arrival.get("prepared_at"))) + parts.append(" (history fetched ") + parts.append(_safe_iso(arrival.get("history_fetched_at"))) + parts.append(", latest state ") + parts.append(_safe_iso(arrival.get("latest_history_state_at"))) + parts.append(", departure observed ") + parts.append(_safe_iso(arrival.get("departure_observed_at"))) + parts.append(")") + parts.append("
") + parts.append("Hypothetical demand: empty-car scenario (not a vehicle reading), ") + parts.append(escape("{} kWh requested, {} kWh at site AC, {} kWh shortfall".format(_format_finite(arrival.get("requested_target_kwh"), 2), _format_finite(arrival.get("model_ac_kwh"), 2), _format_finite(arrival.get("shortfall_kwh"), 2)))) + parts.append("
") + parts.append("Ready by: ") + parts.append(_safe_iso(arrival.get("ready_by"))) + parts.append(" (guaranteed target met: ") + parts.append(escape(str(arrival.get("guaranteed_target_met")))) + parts.append(", execution permission: ") + parts.append(escape(str(arrival.get("execution_permission")))) + parts.append(")") + else: + parts.append("
") + parts.append("Reason: ") + parts.append(escape(str(arrival.get("reason", "")))) + if arrival.get("prepared_at") is not None: + parts.append("
") + parts.append("Source timestamp: ") + parts.append(_safe_iso(arrival.get("prepared_at"))) + parts.append("") + return "".join(parts) + + +def render_joint_site_preview(report, native_diagnostic, now=None): + """Render only the ownership/freshness banner above the existing Plan table. + + The visible mainline never claims physical EV success: only the current + command observed by the executor is reflected in the rows; the future + forecast remains under revalidation. ISO timestamps live under Show + Debug, not in the mainline. + + For shadow / off / error / unavailable candidates the banner labels the + table as a read-only simulation and the compact "Simulation only — not + executed" line is preserved. For a valid applied display (paired rows + + fresh receipt + no stale-SoC/counterfactual hypothesis + coherent + ``generated_at``) the banner switches to "Active plan" and the + simulated-only line is suppressed. For an applied-but-invalid receipt + (expired / missing / future / false authority / blocked / missing + preview / stale ``generated_at``) the compact carries a bounded + execution-blocked reason. Parent ``status='apply_blocked'`` reports + take the same execution-blocked branch. The native-grid-limit diagnostic + remains visible only when no applied receipt is in force; once applied, + the obsolete counterfactual is moved into the Show-Debug detail span. + """ + if not isinstance(report, dict) and not isinstance(native_diagnostic, dict): + return "" + now = now if now is not None else datetime.now(timezone.utc) + if isinstance(report, dict) and report.get("status") == "household_fallback": + return _household_fallback_debug(report) + active = isinstance(report, dict) and report.get("status") != "disabled" + status = report.get("status", "") if isinstance(report, dict) else "" + is_applied = status == "applied" + is_blocked = status == "apply_blocked" + paired = _paired_preview(report, now) if active else None + applied_fresh = _applied_display_allowed(report, now, paired) + compact = [] + debug_parts = [] + if paired is not None: + if applied_fresh: + compact.append("Active plan") + compact.append("current command only — future forecast under revalidation") + if report.get("partial_target_authorised") is True: + compact.append("EV {}% / {}% requested".format(_format_finite(report.get("planned_ev_percent_by_departure"), 1), _format_finite(report.get("requested_ev_target_percent"), 1))) + elif is_applied or is_blocked: + compact.append("execution blocked: {}".format(_applied_block_reason(report, now))) + else: + compact.append("Simulation only — not executed") + if status in ("partial_candidate", "conditional_partial_candidate"): + compact.append("EV {}% / {}% requested".format(_format_finite(report["planned_ev_percent_by_departure"], 1), _format_finite(report["requested_ev_target_percent"], 1))) + if report.get("uses_stale_soc_assumption") or str(status).startswith("conditional_"): + compact.append("EV SOC unconfirmed") + if _is_exceeded(*_parse_native_diag(native_diagnostic)): + compact.append("native grid limit exceeded") + compact.append("Details: Show Debug") + elif (is_applied or is_blocked) and active: + compact.append("execution blocked: {}".format(_applied_block_reason(report, now))) + if applied_fresh: + banner = ( + '
Active plan: ' + '
" + ) + elif (is_applied or is_blocked) and active: + banner = ( + '
Execution blocked: ' + '
" + ) + else: + banner = ( + '
Joint candidate — read-only simulation: ' + '
" + if active + else '
Native plan view: the existing table shows the native schedule, not a joint candidate.
' + ) + state, peak_w, minutes_over, limit_w = _parse_native_diag(native_diagnostic) + native_exceeded = _is_exceeded(state, peak_w, minutes_over, limit_w) + parts = ['
'] + if compact: + parts.append('
' + escape(" · ".join(compact)) + "
") + parts.append('
") + return "".join(parts) + if (is_applied or is_blocked) and not applied_fresh: + # Stale-SoC and conditional labels are only meaningful for shadow / + # partial candidates; an applied or apply_blocked report is handled + # by the compact line above. + parts.append('
Report is in an applied/execution-blocked state. No simulated candidate is shown.
') + if debug_parts: + parts.append('") + debug_parts = [] + if compact: + parts.append("") + parts.append("") + return "".join(parts) + if not applied_fresh and (bool(report.get("uses_stale_soc_assumption")) or (isinstance(status, str) and status.startswith("conditional_"))): + parts.append('
EV SOC is old: stale SoC assumption, conditional simulation, never valid for execution.
') + age = _age_minutes(report.get("generated_at"), now) + if age is not None and (age > _STALE_AGE_MINUTES or age < -1): + parts.append('
' "Report is older than 15 minutes; treating as stale and showing " "no earlier schedule (future timestamps are also invalid).
") + return "".join(parts) + if age is None: + parts.append('
' "Report timestamp is missing or malformed; no candidate is shown.
") + return "".join(parts) + if active and paired is None and not (is_applied or is_blocked): + parts.append("

" + escape(_unavailable_reason(report)) + "

") + elif status in ("partial_candidate", "conditional_partial_candidate") and not applied_fresh: + parts.append( + '".format( + planned=_format_finite(report["planned_ev_percent_by_departure"], 1), + requested=_format_finite(report["requested_ev_target_percent"], 1), + departure=escape(report["ev_departure_at"]), + ) + ) + if debug_parts: + parts.append('") + if compact: + parts.append("") + parts.append("") + return "".join(parts) + + +def get_joint_preview_update_js(): + """No second poller: the existing Plan refresh now updates banner and rows together.""" + return "" diff --git a/apps/predbat/joint_site_recurring.py b/apps/predbat/joint_site_recurring.py new file mode 100644 index 000000000..67028fb41 --- /dev/null +++ b/apps/predbat/joint_site_recurring.py @@ -0,0 +1,385 @@ +"""Opt-in daily EV intent with sensor-derived sessions and durable metering evidence. + +Standing user intent is not a daily operator confirmation or an EV execution lease. +Only the existing joint executor can consume the resulting bounded planning session. +""" + +from copy import deepcopy +from datetime import datetime, timedelta, timezone +import hashlib +import json +from threading import RLock +from zoneinfo import ZoneInfo + + +_LOCK = RLock() + + +def recurring_enabled(settings): + """Recognise explicit opt-in without reading entities or changing legacy behaviour.""" + return isinstance(settings, dict) and settings.get("mode") == "apply" and isinstance(settings.get("recurring_session"), dict) and settings["recurring_session"].get("enabled") is True + + +def next_departure(base, now): + """Use the native ready time on the next local calendar day, including DST changes.""" + ready = datetime.strptime(base.car_charging_plan_time[0], "%H:%M:%S").time() + zone = ZoneInfo(base.args["timezone"]) + local = now.astimezone(zone) + date = local.date() + departure = datetime.combine(date, ready, tzinfo=zone) + if departure <= local: + departure = datetime.combine(date + timedelta(days=1), ready, tzinfo=zone) + return departure.astimezone(timezone.utc) + + +def _save(storage, before, after): + """Require the real Storage acknowledgement; a memory mutation is not persistence.""" + from ha import run_async + + if before != after and run_async(storage.save("joint_site", "recurring_session", after, format="json", expiry=None)) is not True: + raise ValueError("recurring_storage_write_failed") + + +def _record_valid(record, now): + """Validate persisted evidence before it can grant or retain any session authority.""" + from joint_site_native import _number, _parse_entity_timestamp + + if not isinstance(record, dict) or type(record.get("version")) is not int or record["version"] != 1 or type(record.get("active")) is not bool or not isinstance(record.get("policy"), str) or not isinstance(record.get("invalid"), str): + raise ValueError("recurring_storage_corrupt") + if record.get("session") is None: + return + session = record["session"] + required = { + "identity", + "started_at", + "home_transition", + "connection_checked_at", + "initial_kwh", + "meter_baseline", + "soc_observed_at", + "last_energy", + "last_checked", + "last_energy_progress_at", + "departure", + "assessment", + "anchor_kwh", + "anchor_energy", + "anchor_soc_at", + "last_vehicle_value", + } + if not isinstance(session, dict) or set(session) != required or not isinstance(session["identity"], str): + raise ValueError("recurring_storage_corrupt") + for name in ("started_at", "home_transition", "connection_checked_at", "soc_observed_at", "last_checked", "last_energy_progress_at", "anchor_soc_at"): + if _parse_entity_timestamp(session[name], name) > now: + raise ValueError("recurring_storage_future") + _parse_entity_timestamp(session["departure"], "departure") + for name in ("initial_kwh", "meter_baseline", "last_energy", "anchor_kwh", "anchor_energy", "last_vehicle_value"): + _number(session[name], name) + if not isinstance(session["assessment"], dict): + raise ValueError("recurring_storage_corrupt") + + +def _close_deadline(record, now, unavailable=False): + """Preserve the previous deadline's own evidence, never later energy or an absent SOC.""" + from joint_site_native import _parse_entity_timestamp + + session = record.get("session") + if not session or now < _parse_entity_timestamp(session["departure"], "previous_departure"): + return + if (record.get("latest_deadline") or {}).get("departure") == session["departure"]: + return + previous = session["assessment"] + met = True if not unavailable and previous.get("observed_target_met") is True else None + record["latest_deadline"] = {"departure": session["departure"], "last_evidence": deepcopy(previous), "met": met, "unavailable": unavailable} + + +def _home_history_continuous(base, entity, since, transition, now): + """Reconcile a cold-cache heartbeat against Recorder, never against refreshed hardware. + + The exact retained error and session are checked by the caller. History must + cover the anchor and the current publication, with only home states between. + Cache failed reads briefly so a missing recorder is not queried every tick. + """ + from joint_site_native import _parse_entity_timestamp + + key = (entity, since.isoformat(), transition.isoformat()) + cached = getattr(base, "joint_site_home_history_check", None) + if cached and cached["key"] == key and 0 <= (now - cached["at"]).total_seconds() < 60: + return cached["continuous"] + continuous = False + try: + history = base.ha_interface.get_history_window(entity, since, now) + if not isinstance(history, list) or len(history) != 1 or not isinstance(history[0], list) or not history[0]: + return False + stamps = [] + for item in history[0]: + if item.get("entity_id") != entity or item.get("state") != "home": + return False + changed = _parse_entity_timestamp(item["last_changed"], "home_history_changed") + updated = _parse_entity_timestamp(item["last_updated"], "home_history_updated") + if changed > updated or updated > now or (stamps and updated < stamps[-1]): + return False + stamps.append(updated) + continuous = stamps[0] <= since and stamps[-1] >= transition + return continuous + except Exception: + return False + finally: + base.joint_site_home_history_check = {"key": key, "at": now, "continuous": continuous} + + +def _acknowledged_recurring_stop(base, settings, record, home, connection, now): + """Recognise an operator-reconciled interruption of one existing physical session. + + This is not standing permission to ignore charging errors. A new publication, + session, departure, genuine home transition or contrary cached history + loses the exception. Reconciliation never supplies a new SOC or meter value. + """ + from joint_site_native import _parse_entity_timestamp + + ack = settings.get("connection_error_acknowledgement") + session = record.get("session") + if not isinstance(ack, dict) or set(ack) != {"state", "observed_at", "acknowledged_at", "departure", "session_identity"} or not session: + return False + observed = _parse_entity_timestamp(ack["observed_at"], "stop_observed_at") + accepted = _parse_entity_timestamp(ack["acknowledged_at"], "stop_acknowledged_at") + departure = _parse_entity_timestamp(ack["departure"], "stop_departure") + if ack["state"] != "4" or ack["session_identity"] != session["identity"] or connection["state"] != "4" or connection["last_updated"] != observed: + return False + if not _parse_entity_timestamp(session["started_at"], "session_start") <= observed <= accepted <= now < departure: + return False + if departure != next_departure(base, now) or departure != _parse_entity_timestamp(session["departure"], "session_departure"): + return False + transition = home.get("value_last_changed", home["last_changed"]) + home_anchor = max(accepted, _parse_entity_timestamp(session["home_transition"], "home_transition")) + if transition > home_anchor: + if any(item["state"] != "home" and _parse_entity_timestamp(item["at"], "home_history") > home_anchor for item in home.get("value_history") or []): + return False + if not _home_history_continuous(base, settings["home_entity"], home_anchor, transition, now): + return False + checked = _parse_entity_timestamp(session["connection_checked_at"], "connection_checked_at") + for item in connection.get("value_history") or []: + stamp = _parse_entity_timestamp(item["at"], "connection_history") + if stamp > checked and item["state"] not in settings["connection_entity_states"] and not (item["state"] == "4" and stamp == observed): + return False + return True + + +def _resolve(base, settings, now): + """Resolve cached observations, reconciling cold home continuity against Recorder if needed.""" + from ha import run_async + from joint_site_native import _number, _parse_entity_timestamp, _parse_soc_raw, _read_entity_state + + config = settings["recurring_session"] + if set(config) != {"enabled", "policy_authorised_at", "energy_entity", "power_entity"}: + raise ValueError("recurring_config_invalid") + authorised = _parse_entity_timestamp(config["policy_authorised_at"], "policy_authorised_at") + if authorised > now or settings.get("mode") != "apply" or settings.get("control_model") != "deye_solarman" or not settings.get("grid_charge_support"): + raise ValueError("recurring_policy_invalid") + if settings.get("execution_authorised") is not True or settings.get("household_fallback") is not True: + raise ValueError("recurring_not_authorised") + if any(not isinstance(config[name], str) or not config[name].startswith("sensor.") for name in ("energy_entity", "power_entity")): + raise ValueError("recurring_meter_mapping_invalid") + components = getattr(base, "components", None) + storage = components.get_component("storage") if components else None + if storage is None: + raise ValueError("recurring_storage_required") + before = run_async(storage.load_checked("joint_site", "recurring_session")) + if before is not None: + _record_valid(before, now) + else: + legacy = run_async(storage.load_checked("joint_site", "confirmed_session_guard")) + if legacy is not None and (not isinstance(legacy, dict) or legacy.get("invalid") is not False): + raise ValueError("recurring_legacy_session_requires_reconciliation") + policy = hashlib.sha256(json.dumps({"config": config, "home": settings.get("home_entity"), "connection": settings.get("connection_entity"), "soc": settings.get("soc_entity")}, sort_keys=True).encode()).hexdigest() + if before is not None and before["policy"] != policy: + raise ValueError("recurring_policy_changed_requires_reconciliation") + record = deepcopy(before) if before is not None else {"version": 1, "policy": policy, "active": False, "invalid": "", "session": None, "latest_deadline": None} + + def read(entity): + """Validate actual cached timestamps, never substitute the planning clock.""" + value = _read_entity_state(base, entity) + if value is None or value["last_changed"] > now or value["last_updated"] > now: + raise ValueError("recurring_entity_unavailable_or_future") + return value + + def unavailable(reason, end_session=True): + """Retain invalidation and close availability without reading any vehicle energy.""" + if end_session: + record["active"] = False + _close_deadline(record, now, unavailable=True) + _save(storage, before, record) + return {"status": "blocked", "reason": reason, "latest_deadline": record["latest_deadline"]} + + # Availability always precedes SOC and the EV meter, including resumed sessions. + try: + home = read(settings.get("home_entity")) + except ValueError: + return unavailable("ev_not_home") + if home["state"] != "home": + return unavailable("ev_not_home") + try: + connection = read(settings.get("connection_entity")) + except ValueError: + return unavailable("connection_not_confirmed") + allowed = settings.get("connection_entity_states") + if not isinstance(allowed, (list, tuple)) or not allowed or connection["attributes"].get("retrievalstatus") != "VALID": + return unavailable("connection_not_confirmed") + acknowledged = _acknowledged_recurring_stop(base, settings, record, home, connection, now) if connection["state"] not in allowed else False + if connection["state"] not in allowed and not acknowledged: + return unavailable("connection_not_confirmed") + try: + permission = read(settings.get("permission_entity")) + except ValueError: + return unavailable("permission_not_granted", end_session=False) + if permission["state"] != "on": + return unavailable("permission_not_granted", end_session=False) + if record["invalid"]: + return unavailable(record["invalid"], end_session=False) + if getattr(base, "car_charging_manual_soc", None) and base.car_charging_manual_soc[0]: + return unavailable("recurring_manual_soc_override", end_session=False) + capacity = _number(base.car_charging_battery_size[0], "capacity", 0.001) + rate = _number(base.car_charging_rate[0], "rate", 0.001) + efficiency = _number(base.car_charging_loss, "efficiency", 0.001, 1) + target = _number(base.car_charging_limit[0], "target", 0, capacity) + departure = next_departure(base, now) + relay = read(settings.get("relay_entity")) + if relay["state"] not in ("off", "on"): + return unavailable("recurring_relay_unavailable", end_session=False) + soc = read(settings.get("soc_entity")) + meter = read(config["energy_entity"]) + power = read(config["power_entity"]) + if soc["attributes"].get("retrievalstatus") != "VALID" or meter["attributes"].get("unit_of_measurement") != "kWh" or power["attributes"].get("unit_of_measurement") != "kW": + return unavailable("recurring_observation_invalid", end_session=False) + measured = _parse_soc_raw(soc["state"], settings.get("soc_entity_unit", "kWh"), capacity) + if measured is None or not 0 <= measured <= capacity or isinstance(meter["state"], bool) or isinstance(power["state"], bool): + return unavailable("recurring_observation_invalid", end_session=False) + total = _number(float(meter["state"]), "meter") + power_kw = _number(float(power["state"]), "power", 0, rate * 1.1) + session = record["session"] + transition = home.get("value_last_changed", home["last_changed"]) + new_home = not acknowledged and session is not None and transition > _parse_entity_timestamp(session["home_transition"], "home_transition") + new_connection = False + if session is not None: + if total < session["last_energy"]: + record["invalid"] = "recurring_meter_contradiction" + return unavailable(record["invalid"], end_session=False) + checked = _parse_entity_timestamp(session["connection_checked_at"], "connection_checked_at") + if connection["last_updated"] < checked: + return unavailable("recurring_connection_clock_reversed") + if not acknowledged and connection["value_last_changed"] > checked: + history = connection.get("value_history") + # Eligible charging-state changes are not new plugs. A cached history + # containing an unplug, or an unobserved gap, requires a new baseline. + if not history or _parse_entity_timestamp(history[0]["at"], "connection_history") > checked: + new_connection = True + else: + new_connection = any(_parse_entity_timestamp(item["at"], "connection_history") > checked and item["state"] not in allowed for item in history) + new_session = (not record["active"] and not acknowledged) or session is None or new_home or new_connection + _close_deadline(record, now, unavailable=new_session) + if new_session: + if relay["state"] != "off" or power_kw > 0.01: + return unavailable("recurring_new_baseline_requires_off", end_session=False) + if measured >= target and (now - soc["last_updated"]).total_seconds() > 180: + return unavailable("recurring_stale_full_soc", end_session=False) + stamp = now.isoformat() + session = { + "identity": hashlib.sha256((policy + stamp).encode()).hexdigest(), + "started_at": stamp, + "home_transition": transition.isoformat(), + "connection_checked_at": connection["last_updated"].isoformat(), + "initial_kwh": measured, + "meter_baseline": total, + "soc_observed_at": soc["last_updated"].isoformat(), + "last_energy": total, + "last_checked": stamp, + "last_energy_progress_at": stamp, + "departure": departure.isoformat(), + "assessment": {}, + "anchor_kwh": measured, + "anchor_energy": total, + "anchor_soc_at": soc["last_changed"].isoformat(), + "last_vehicle_value": measured, + } + record.update(active=True, session=session) + initial = _number(session["initial_kwh"], "initial", 0, capacity) + started = _parse_entity_timestamp(session["started_at"], "session_start") + elapsed = (now - started).total_seconds() / 3600 + delta = total - session["meter_baseline"] + anchor_delta = total - session["anchor_energy"] + progress_at = _parse_entity_timestamp(session["last_energy_progress_at"], "last_energy_progress_at") + recent_hours = (meter["last_changed"] - progress_at).total_seconds() / 3600 + progressed = total > session["last_energy"] + invalid = "" + if delta < 0 or anchor_delta < 0 or anchor_delta > capacity / efficiency + 0.5 or delta > rate * elapsed * 1.1 + 0.1 or (progressed and (recent_hours <= 0 or total - session["last_energy"] > rate * recent_hours * 1.1 + 0.1)): + invalid = "recurring_meter_contradiction" + estimate = min(capacity, session["anchor_kwh"] + max(0, anchor_delta) * efficiency) + changed = soc["last_changed"] > started and abs(measured - initial) > 0.001 + recent = changed and (now - soc["last_changed"]).total_seconds() <= 180 + tolerance = max(0.5, capacity * 0.05) + if changed and (measured < initial - tolerance or measured > estimate + tolerance or (recent and abs(measured - estimate) > tolerance)): + invalid = "recurring_soc_contradiction" + if not invalid and recent and measured != session["last_vehicle_value"] and soc["last_changed"] > _parse_entity_timestamp(session["anchor_soc_at"], "anchor_soc_at"): + session.update(anchor_kwh=measured, anchor_energy=total, anchor_soc_at=soc["last_changed"].isoformat()) + completed_observation = changed and measured >= target + value = measured if recent or completed_observation else estimate + if not invalid and progressed: + progress_at = meter["last_changed"] + session["last_energy_progress_at"] = progress_at.isoformat() + if value < target and relay["state"] == "on" and min((now - relay["last_changed"]).total_seconds(), (now - progress_at).total_seconds()) > 600: + invalid = "recurring_meter_stopped" + if invalid: + record["invalid"] = invalid + return unavailable(invalid, end_session=False) + session["departure"] = departure.isoformat() + observation = { + "ev_soc": value, + "ev_soc_source": "changed_vehicle_value_under_standing_policy" if recent or completed_observation else "standing_policy_meter_estimate", + "soc_observed_at": soc["last_updated"].isoformat(), + "soc_observation_age_minutes": (now - soc["last_updated"]).total_seconds() / 60, + "ev_soc_is_estimate": not (recent or completed_observation), + "ev_metered_ac_kwh": delta, + "vehicle_value_kwh": measured, + "policy_authorised_at": config["policy_authorised_at"], + "sensor_session_started_at": session["started_at"], + "observed_target_met": bool(completed_observation), + "connection_error_acknowledged": acknowledged, + "connection_error_observed_at": connection["last_updated"].isoformat() if acknowledged else None, + } + session.update( + last_energy=total, + last_checked=now.isoformat(), + connection_checked_at=connection["last_updated"].isoformat(), + last_vehicle_value=measured, + assessment={"at": now.isoformat(), "soc_kwh": value, "shortfall_kwh": max(0, target - value), "is_estimate": observation["ev_soc_is_estimate"], "observed_target_met": observation["observed_target_met"]}, + ) + if acknowledged: + record["active"] = True + session["home_transition"] = transition.isoformat() + _save(storage, before, record) + return { + "status": "ready", + "reason": None, + "identity": session["identity"], + "session": {"available_from": session["started_at"], "departure": session["departure"], "soc_kwh": value, "soc_observed_at": soc["last_updated"].isoformat()}, + "observation": observation, + "latest_deadline": record["latest_deadline"], + } + + +def resolve_recurring_session(base, settings=None, now=None): + """Resolve standing intent without mutating user configuration or writing to actuators.""" + from joint_site_native import _observation_clock + + settings = base.args.get("joint_site") if settings is None else settings + if not recurring_enabled(settings): + return {"status": "disabled", "reason": None} + now = _observation_clock() if now is None else now + try: + if not isinstance(now, datetime) or now.utcoffset() is None: + raise ValueError("recurring_clock_invalid") + with _LOCK: + return _resolve(base, settings, now.astimezone(timezone.utc)) + except Exception as exc: + reason = str(exc) if isinstance(exc, ValueError) and str(exc).startswith("recurring_") else "recurring_evidence_invalid" + return {"status": "blocked", "reason": reason} diff --git a/apps/predbat/joint_site_replay.py b/apps/predbat/joint_site_replay.py new file mode 100644 index 000000000..a1eb4625f --- /dev/null +++ b/apps/predbat/joint_site_replay.py @@ -0,0 +1,95 @@ +"""Segmented native replay for explicit per-period battery discharge ceilings. + +The existing Prediction engines and kernel ABI are unchanged. Each five-minute +step is a new native Python Prediction with the current-rate input set explicitly +for limited demand. This is an independent check, not an actuator implementation. +""" + +from copy import copy +from datetime import timedelta +import math +from types import SimpleNamespace + +from const import MINUTE_WATT, PREDICT_STEP +from prediction import Prediction + + +def discharge_limit_w(base, action): + """Convert a proposed AC ceiling to whole native DC watts without raising a cap.""" + limit = getattr(action, "discharge_limit_kw", None) + if limit is None: + return None + return min(int(base.battery_rate_max_discharge * MINUTE_WATT), math.ceil(limit * 1000 / base.inverter_loss)) + + +def replay_discharge_limits(model, result, slots, scenario): + """Replay a complete trajectory through unmodified native step calculations. + + Native final SOC is rounded to four decimals. Track the worst accumulated + rounding uncertainty so acceptance checks can retain a conservative margin. + The original base, forecasts, active Prediction and kernel remain untouched. + """ + soc = model.soc_kw + car_soc = list(model.car_charging_soc) + trace = {} + grid = {} + cost = 0 + peak = 0 + minutes_over = 0 + checked = 0 + for action, slot in zip(result.actions, slots): + if slot.duration_min != PREDICT_STEP: + raise ValueError("Native discharge replay requires five-minute steps") + step_base = copy(model) + step_base.minutes_now = model.minutes_now + slot.start_min + step_base.now_utc = model.midnight_utc + timedelta(minutes=step_base.minutes_now) + step_base.forecast_minutes = PREDICT_STEP + step_base.soc_kw = soc + step_base.car_charging_soc = list(car_soc) + step_base.cost_today_sofar = 0 + step_base.battery_temperature_prediction = {0: model.battery_temperature_prediction.get(slot.start_min, model.battery_temperature)} + trace[slot.start_min] = soc + power = discharge_limit_w(model, action) + if power is not None: + # No charge/export window applies in limited demand. Suppress only + # the native per-step rate reset, retaining original curve ceilings. + step_base.set_charge_window = False + step_base.set_export_window = False + step_base.discharge_rate_now = power / MINUTE_WATT + step_base.charge_rate_now = model.battery_rate_max_charge + windows = [] + limits = [] + if action.battery_mode in ("charge", "hold"): + windows = [{"start": step_base.minutes_now, "end": step_base.minutes_now + PREDICT_STEP, "average": slot.import_price}] + limits = [model.soc_max if action.battery_mode == "charge" else model.reserve] + pred = Prediction( + step_base, + {0: model.pv_forecast_minute_step[slot.start_min]}, + {0: model.pv_forecast_minute10_step[slot.start_min]}, + {0: model.load_minutes_step[slot.start_min]}, + {0: model.load_minutes_step10[slot.start_min]}, + ) + pred.run_prediction(limits, windows, [], [], scenario, end_record=PREDICT_STEP, save="test") + soc = pred.final_soc + car_soc = list(pred.final_car_soc) + cost += pred.final_metric + grid.update(pred.predict_grid_power) + diagnostic = getattr(pred, "import_limit_diagnostics", None) + if diagnostic and diagnostic["checked_minutes"] == PREDICT_STEP: + peak = max(peak, diagnostic["peak_w"]) + minutes_over += diagnostic["minutes_over"] + checked += diagnostic["checked_minutes"] + else: + values = list(pred.predict_grid_power.values()) + peak = max(peak, max((max(0, value * 1000 + 0.5) for value in values), default=0)) + minutes_over += sum(value * 1000 + 0.5 > model.import_limit for value in values) * PREDICT_STEP + checked += len(values) * PREDICT_STEP + return SimpleNamespace( + predict_soc=trace, + predict_grid_power=grid, + final_soc=soc, + final_car_soc=car_soc, + final_metric=cost + model.cost_today_sofar, + import_limit_diagnostics={"peak_w": peak, "minutes_over": minutes_over, "checked_minutes": checked}, + rounding_bound_kwh=len(slots) * 0.00005, + ) diff --git a/apps/predbat/joint_site_session.py b/apps/predbat/joint_site_session.py new file mode 100644 index 000000000..da5dd0a1b --- /dev/null +++ b/apps/predbat/joint_site_session.py @@ -0,0 +1,144 @@ +"""EV availability routing and explicit partial-target evidence, never actuator control.""" + +import hashlib +import json +import math + + +def household_reason(base): + """Check availability BEFORE reading vehicle SOC or the confirmed-session meter.""" + settings = base.args.get("joint_site") + if not isinstance(settings, dict) or settings.get("mode") != "apply" or settings.get("household_fallback") is not True: + return None + from joint_site_recurring import recurring_enabled, resolve_recurring_session + + if recurring_enabled(settings): + return resolve_recurring_session(base)["reason"] + from joint_site_native import _read_entity_state, _observation_clock, _parse_entity_timestamp + + now = _observation_clock() + + def observed(name): + """Require a cached, non-future observation; never refresh a device.""" + value = _read_entity_state(base, settings.get(name)) + if value is None or value["last_changed"] > now or value["last_updated"] > now: + return None + return value + + if settings.get("home_entity"): + home = observed("home_entity") + if home is None or home["state"] != "home": + return "ev_not_home" + connection = observed("connection_entity") + allowed = settings.get("connection_entity_states") + if connection is None: + return "connection_not_confirmed" + if not isinstance(allowed, (list, tuple)) or connection["state"] not in allowed: + from joint_site_apply import _acknowledged_stop_state + + try: + acknowledged = _acknowledged_stop_state(settings, connection, now) + except (ValueError, TypeError, KeyError): + acknowledged = False + if not acknowledged: + return "connection_not_confirmed" + permission = observed("permission_entity") + if permission is None or permission["state"] != "on": + return "permission_not_granted" + try: + session = settings["session"] + departure = _parse_entity_timestamp(session["departure"], "departure") + arrival = _parse_entity_timestamp(session["available_from"], "arrival") + if now >= departure: + return "departure_passed" + if arrival > now or arrival >= departure: + return "ev_outside_session" + except (ValueError, TypeError, KeyError): + return "ev_session_required" + authorised = settings.get("execution_authorised") is True and settings.get("confirmed_session") and settings.get("grid_charge_support") and settings.get("control_model") == "deye_solarman" + if settings.get("apply_verified") is not True and not authorised: + return "apply_commissioning_required" + confirmation = settings.get("confirmed_session") + if confirmation: + try: + confirmed = _parse_entity_timestamp(confirmation["confirmed_at"], "confirmed_at") + if settings.get("home_entity") and home["value_last_changed"] > confirmed: + return "new_vehicle_session_required" + relay = observed("relay_entity") + if relay and relay["state"] == "off" and connection["value_last_changed"] > confirmed and connection["state"] in (allowed or []): + return "new_connection_confirmation_required" + except (ValueError, TypeError, KeyError): + return "ev_session_required" + return None + + +def partial_metadata_valid(base, report): + """Require finite, consistent requested/planning/delivery evidence from both native replays.""" + settings = base.args.get("joint_site", {}) + if settings.get("ev_partial_target_authorised") is not True or settings.get("control_model") != "deye_solarman" or not settings.get("grid_charge_support"): + return False + fields = ("requested_ev_target_kwh", "planning_ev_target_kwh", "planned_ev_kwh_by_departure", "ev_target_shortfall_kwh") + if any(type(report.get(key)) not in (int, float) or not math.isfinite(report[key]) for key in fields): + return False + requested, planning, delivered, shortfall = (report[key] for key in fields) + initial = report.get("ev_initial_state", {}).get("ev_soc_kwh") + if type(initial) not in (int, float) or not math.isfinite(initial): + return False + if requested != base.car_charging_limit[0] or not 0 <= initial < planning < requested or not planning - 0.002 <= delivered < requested - 0.002: + return False + if report.get("requested_ev_target_met") is not False or report.get("partial_target_authorised") is not True or abs(shortfall - (requested - delivered)) > 0.002: + return False + checks = report.get("native_checks", {}) + if set(checks) != {"nominal", "pv10"}: + return False + for check in checks.values(): + target, final = check.get("validated_ev_target_kwh"), check.get("final_ev_kwh") + if type(target) not in (int, float) or type(final) not in (int, float) or not math.isfinite(target) or not math.isfinite(final): + return False + if target != planning or final < planning - 0.002 or check.get("passed") is not True or check.get("checked_minutes") != base.forecast_minutes: + return False + return abs(delivered - min(check["final_ev_kwh"] for check in checks.values())) <= 0.002 + + +def partial_fingerprint(report): + """Bind targets and paired evidence to the staged candidate, separately from user configuration.""" + keys = ("requested_ev_target_kwh", "planning_ev_target_kwh", "planned_ev_kwh_by_departure", "ev_target_shortfall_kwh", "requested_ev_target_met", "partial_target_authorised", "ev_initial_state") + payload = {key: report.get(key) for key in keys} + payload["checks"] = {name: {key: check.get(key) for key in ("passed", "checked_minutes", "validated_ev_target_kwh", "final_ev_kwh")} for name, check in report.get("native_checks", {}).items()} + return hashlib.sha256(json.dumps(payload, sort_keys=True, allow_nan=False).encode()).hexdigest() + + +def retain_session_validity(base, settings, contradiction, soc_changed_at, now): + """Persist invalidation through Storage; only a strictly newer explicit confirmation can replace it.""" + if settings.get("mode") != "apply" or settings.get("household_fallback") is not True: + if contradiction: + raise ValueError("EV SOC contradicts confirmed metered delivery") + return + from ha import run_async + from joint_site_native import _parse_entity_timestamp + + components = getattr(base, "components", None) + storage = components.get_component("storage") if components else None + if storage is None: + raise ValueError("Confirmed session requires durable validation storage") + config = settings["confirmed_session"] + identity = hashlib.sha256(json.dumps({"confirmation": config, "soc_entity": settings.get("soc_entity"), "connection_entity": settings.get("connection_entity")}, sort_keys=True).encode()).hexdigest() + confirmed = _parse_entity_timestamp(config["confirmed_at"], "confirmed_at") + if confirmed > now: + raise ValueError("Confirmed session is future-dated") + record = run_async(storage.load("joint_site", "confirmed_session_guard")) + if record is not None: + if not isinstance(record, dict) or set(record) != {"identity", "confirmed_at", "invalid", "soc_changed_at"} or type(record["invalid"]) is not bool: + raise ValueError("Confirmed session validity record is invalid") + prior = _parse_entity_timestamp(record["confirmed_at"], "previous_confirmation") + if record["identity"] == identity: + if record["invalid"]: + raise ValueError("Confirmed EV session requires explicit reconciliation") + elif confirmed <= prior: + raise ValueError("A new explicit confirmation is required") + updated = {"identity": identity, "confirmed_at": config["confirmed_at"], "invalid": bool(contradiction), "soc_changed_at": soc_changed_at.isoformat()} + if record != updated: + if run_async(storage.save("joint_site", "confirmed_session_guard", updated, format="json", expiry=None)) is not True: + raise ValueError("Confirmed session validation could not be persisted") + if contradiction: + raise ValueError("EV SOC contradicts confirmed metered delivery") diff --git a/apps/predbat/joint_site_template.py b/apps/predbat/joint_site_template.py new file mode 100644 index 000000000..fcef129ad --- /dev/null +++ b/apps/predbat/joint_site_template.py @@ -0,0 +1,297 @@ +# ----------------------------------------------------------------------------- +# Predbat Home Battery System +# Copyright Trefor Southwell 2026 - All Rights Reserved +# This application maybe used for personal use only and not for commercial use +# ----------------------------------------------------------------------------- +"""Bounded adapter for the shared Deye/Solarman verified-service template. + +Three pure helpers - no HA service writes of the adapter's own: + +* :func:`validate_template` resolves the canonical six-row charge and + demand sequences through :meth:`Inverter.prepare_service_sequence` + and compares the resulting entities, services and per-phase values + against the canonical mapping. Returns ``True`` on valid; raises + :class:`ServiceSequenceError` (a :class:`ValueError` subclass) on + any mismatch so a malicious mapping that targets another inverter + cannot pass preflight only because the cached terminal state already + matches. +* :func:`snapshot_template` reads every canonical row from the HA + cache via :meth:`Inverter.get_state_wrapper`, applies + :func:`math.ceil` to the Predbat reserve and returns the resolved + configuration mapping plus the current control states. +* :func:`dispatch_template` calls + :meth:`Inverter.adjust_charge_immediate` exactly once with + ``(target_soc, freeze=False)`` and re-observes the final six-row + state through :meth:`Inverter.service_sequence_state_matches`. + Returns the literal ``bool`` verdict: ``True`` only when every + canonical row reaches the expected terminal state and + ``base.had_errors`` stays ``False`` across the dispatch. + +Two default intents - ``demand(0, False)`` and ``charge(100, False)``. +Explicit ``charge_ceiling=True`` additionally accepts replay-validated integer +targets strictly above reserve. Freeze and discharge-power changes stay unsupported. + +The parent (``joint_site_apply``) owns commissioning, SOC/connection +freshness (≤180 s), the 300 s lease TTL and the EV external executor. +Verified dedup is sequence-based, not hash-based; do not migrate. +""" + +import math +import re + +from const import MINUTE_WATT +import inverter as native_inverter + +# The verified-sequence core is optional until the shared-template PR is installed. +# A missing core must not break imports of the independent joint planner. +ServiceSequenceError = getattr(native_inverter, "ServiceSequenceError", ValueError) + + +PROFILE_ID = "deye_solarman_shared" +EXPECTED_ROW_COUNT = 6 +DISABLED_OPTION = "Disabled" +ENABLE_ON_CHARGE_OPTION = "Grid" + + +def _expected_entities(prefix): + """Return only the canonical six source and SOC entities for one inverter prefix.""" + sources = ["select.{}_program_{}_charging".format(prefix, row) for row in range(1, EXPECTED_ROW_COUNT + 1)] + targets = ["number.{}_program_{}_soc".format(prefix, row) for row in range(1, EXPECTED_ROW_COUNT + 1)] + return sources, targets + + +def _is_real_inverter(inverter): + """``True`` only for an actual :class:`Inverter` instance - never a label string.""" + from inverter import Inverter as _Inverter + + return isinstance(inverter, _Inverter) + + +def _resolve_canonical_steps(inverter): + """Run the native preflight at the canonical dispatch values and return the resolved steps. + + ``charge_start_service`` is resolved with ``target_soc=100`` so any + out-of-bounds placeholder substitution fails closed BEFORE the + adapter ever authorises a write. ``charge_stop_service`` is resolved + with the Predbat ``reserve_percent`` rounded UP through + :func:`math.ceil` exactly the way the canonical ``{reserve_soc}`` + substitution is computed at dispatch time. + """ + base = inverter.base + return ( + inverter.prepare_service_sequence(base.args["charge_start_service"], {"target_soc": 100}, {}), + inverter.prepare_service_sequence(base.args["charge_stop_service"], {"target_soc": 100}, {}), + ) + + +def validate_template(inverter): + """Return ``True`` iff the inverter matches the canonical shared template.""" + if not _is_real_inverter(inverter): + raise ServiceSequenceError("not_an_inverter_instance") + if not all(callable(getattr(inverter, name, None)) for name in ("prepare_service_sequence", "call_service_sequence", "service_sequence_state_matches")): + raise ServiceSequenceError("verified_template_core_required") + base = getattr(inverter, "base", None) + args = getattr(base, "args", None) if base is not None else None + if not isinstance(args, dict): + raise ServiceSequenceError("base_args_missing") + if not isinstance(inverter.inverter_type, str) or inverter.inverter_type != "DeyeSolarman": + raise ServiceSequenceError("inverter_type_must_be_DeyeSolarman") + prefix = args.get("deye_entity_prefix") + if not isinstance(prefix, str) or not re.fullmatch(r"[a-z0-9_]+", prefix): + raise ServiceSequenceError("deye_entity_prefix_invalid") + if args.get("discharge_stop_service") != []: + raise ServiceSequenceError("discharge_stop_service_must_be_empty_list") + if args.get("charge_freeze_service") not in (None, "", []): + raise ServiceSequenceError("charge_freeze_service_unsupported") + + capability_invariants = ( + ("inv_support_charge_freeze", False), + ("inv_support_discharge_freeze", False), + ("inv_has_timed_pause", False), + ("inv_charge_control_immediate", False), + ("inv_has_target_soc", False), + ("inv_has_reserve_soc", False), + ("inv_has_solis_energy_control", False), + ("inv_has_charge_enable_time", False), + ("inv_has_discharge_enable_time", False), + ("inv_charge_discharge_with_rate", False), + ("inv_charge_time_format", "S"), + ("inv_output_charge_control", "none"), + ) + for attr, expected in capability_invariants: + if not hasattr(inverter, attr) or getattr(inverter, attr) != expected: + raise ServiceSequenceError(attr + "_mismatch") + + for name in ("battery_rate_max_charge", "battery_rate_max_charge_dc", "battery_rate_max_discharge", "inverter_limit", "export_limit"): + value = getattr(base, name, None) + if type(value) not in (int, float) or not math.isfinite(value) or (value < 0 if name == "export_limit" else value <= 0): + raise ServiceSequenceError(name + "_must_be_positive") + native_value = getattr(inverter, name, None) + if type(native_value) not in (int, float) or not math.isfinite(native_value) or not math.isclose(native_value, value, rel_tol=0, abs_tol=1e-10): + raise ServiceSequenceError(name + "_native_model_mismatch") + for name in ("battery_rate_max_scaling", "battery_rate_max_scaling_discharge"): + if type(getattr(base, name, None)) not in (int, float) or getattr(base, name) != 1: + raise ServiceSequenceError("template_rate_scaling_unsupported") + if type(inverter.reserve_percent) not in (int, float) or not math.isfinite(inverter.reserve_percent) or not 0 <= inverter.reserve_percent <= 100: + raise ServiceSequenceError("reserve_percent_invalid") + capacity = getattr(base, "soc_max", None) + reserve = getattr(base, "reserve", None) + if type(capacity) not in (int, float) or not math.isfinite(capacity) or capacity <= 0 or not math.isclose(capacity, inverter.soc_max, rel_tol=0, abs_tol=0.001): + raise ServiceSequenceError("template_capacity_model_mismatch") + if type(reserve) not in (int, float) or not math.isfinite(reserve) or not math.isclose(reserve, inverter.soc_max * inverter.reserve_percent / 100, rel_tol=0, abs_tol=0.001): + raise ServiceSequenceError("template_reserve_model_mismatch") + + charge_start = args.get("charge_start_service") + charge_stop = args.get("charge_stop_service") + if not (isinstance(charge_start, list) and len(charge_start) == 3): + raise ServiceSequenceError("charge_start_service_length_must_be_three") + if not (isinstance(charge_stop, list) and len(charge_stop) == 2): + raise ServiceSequenceError("charge_stop_service_length_must_be_two") + for step in (*charge_start, *charge_stop): + if not (isinstance(step, dict) and step.get("verify") is True): + raise ServiceSequenceError("every_step_must_have_verify_true") + if "device_id" in step or step.get("data") or step.get("payload"): + raise ServiceSequenceError("steps_must_not_contain_device_id_or_data_payload") + if not (isinstance(charge_start[1].get("value"), str) and charge_start[1]["value"].strip() == "{target_soc}"): + raise ServiceSequenceError("charge_start_service_target_step_must_carry_target_soc_placeholder") + if charge_start[0].get("option") != DISABLED_OPTION or charge_start[2].get("option") != ENABLE_ON_CHARGE_OPTION: + raise ServiceSequenceError("charge_start_service_source_options_must_be_Disabled_then_Grid") + if charge_stop[0].get("option") != DISABLED_OPTION: + raise ServiceSequenceError("charge_stop_service_disable_option_must_be_Disabled") + if not (isinstance(charge_stop[1].get("value"), str) and charge_stop[1]["value"].strip() == "{reserve_soc}"): + raise ServiceSequenceError("charge_stop_service_target_step_must_carry_reserve_soc_placeholder") + + charge_steps, demand_steps = _resolve_canonical_steps(inverter) + sources, targets = _expected_entities(prefix) + + if len(charge_steps) != 3: + raise ServiceSequenceError("charge_steps_resolved_count_mismatch") + if len(demand_steps) != 2: + raise ServiceSequenceError("demand_steps_resolved_count_mismatch") + if charge_steps[0][0] != "select/select_option" or charge_steps[2][0] != "select/select_option": + raise ServiceSequenceError("charge_phase_service_must_be_select_option") + if charge_steps[1][0] != "number/set_value": + raise ServiceSequenceError("charge_target_phase_service_must_be_number_set_value") + disable_sources = list(charge_steps[0][2]) + target_phase_entities = list(charge_steps[1][2]) + enable_sources = list(charge_steps[2][2]) + if disable_sources != sources or enable_sources != sources: + raise ServiceSequenceError("charge_phase_source_rows_mismatch") + if target_phase_entities != targets: + raise ServiceSequenceError("charge_target_phase_rows_mismatch") + if charge_steps[1][3] != 100.0: + raise ServiceSequenceError("charge_target_phase_resolved_value_mismatch:" + str(charge_steps[1][3])) + if list(demand_steps[0][2]) != sources or demand_steps[0][0] != "select/select_option": + raise ServiceSequenceError("demand_disable_phase_rows_or_service_mismatch") + if list(demand_steps[1][2]) != targets or demand_steps[1][0] != "number/set_value": + raise ServiceSequenceError("demand_target_phase_rows_or_service_mismatch") + expected_demand_target = float(math.ceil(inverter.reserve_percent)) + if demand_steps[1][3] != expected_demand_target: + raise ServiceSequenceError("demand_target_phase_resolved_value_mismatch:" + str(demand_steps[1][3])) + + return True + + +def snapshot_template(inverter): + """Cache-only mirror of the verified six rows plus the resolved configuration mapping.""" + validate_template(inverter) + base = inverter.base + prefix = base.args.get("deye_entity_prefix") + sources, targets = _expected_entities(prefix) + sources_state, targets_state = [], [] + for entity in sources: + payload = base.get_state_wrapper(entity_id=entity, raw=True, refresh=False) + if not isinstance(payload, dict) or payload.get("state") in (None, "unknown", "unavailable", "none", ""): + raise ServiceSequenceError("cache_observation_unavailable:" + entity) + sources_state.append(payload.get("state")) + for entity in targets: + payload = base.get_state_wrapper(entity_id=entity, raw=True, refresh=False) + if not isinstance(payload, dict) or payload.get("state") in (None, "unknown", "unavailable", "none", ""): + raise ServiceSequenceError("cache_observation_unavailable:" + entity) + state = payload.get("state") + if not isinstance(state, (int, float)): + try: + float(str(state)) + except (TypeError, ValueError): + raise ServiceSequenceError("cache_target_state_not_numeric:" + entity) + targets_state.append(state) + reserve_percent = inverter.reserve_percent + if not (isinstance(reserve_percent, (int, float)) and math.isfinite(reserve_percent)): + raise ServiceSequenceError("reserve_percent_invalid") + return { + "profile_id": PROFILE_ID, + "prefix": prefix, + "source_entities": sources, + "target_entities": targets, + "current_sources": sources_state, + "current_targets": targets_state, + "demand_reserve_soc": int(math.ceil(float(reserve_percent))), + "charge_target_soc": 100.0, + "configuration": { + "inverter_type": inverter.inverter_type, + "charge_start_service": base.args["charge_start_service"], + "charge_stop_service": base.args["charge_stop_service"], + "discharge_stop_service": base.args["discharge_stop_service"], + "battery_rate_max_charge_w": base.battery_rate_max_charge * MINUTE_WATT, + "battery_rate_max_discharge_w": base.battery_rate_max_discharge * MINUTE_WATT, + "reserve_percent": float(reserve_percent), + }, + } + + +def dispatch_template(inverter, target_soc, freeze=False, discharge_power_w=None, charge_ceiling=False): + """Drive the verified shared template through :meth:`Inverter.adjust_charge_immediate`. + + Bounded intermediate charge targets require explicit ``charge_ceiling=True``; + defaults keep the demand/full-charge contract. Invalid intents fail before + the inverter is called. Native dispatch calls + ``adjust_charge_immediate(target_soc, freeze=False)`` exactly once + with no signature fallback. + + Returns the literal ``bool`` verdict: ``True`` iff the canonical + preflight passes, ``base.had_errors`` stays ``False`` across the + dispatch and every canonical row reaches the expected terminal + state; ``False`` on any partial observation, had_errors divergence + or native exception. + """ + if freeze is not False: + raise ValueError("freeze_unsupported") + if discharge_power_w is not None: + raise ValueError("discharge_power_w_unsupported") + bounded = charge_ceiling is True and type(target_soc) is int and math.ceil(inverter.reserve_percent) < target_soc <= 100 + if type(target_soc) is not int or (target_soc not in (0, 100) and not bounded): + raise ValueError("unsupported_target_soc") + try: + validate_template(inverter) + except ServiceSequenceError: + return False + + base = inverter.base + if getattr(base, "had_errors", False): + return False + + try: + inverter.adjust_charge_immediate(target_soc, freeze=False) + except Exception: + return False + if getattr(base, "had_errors", False): + return False + + sources, targets = _expected_entities(base.args.get("deye_entity_prefix")) + if target_soc > 0: + expected_source, expected_target = ENABLE_ON_CHARGE_OPTION, float(target_soc) + else: + reserve_percent = inverter.reserve_percent + if not (isinstance(reserve_percent, (int, float)) and math.isfinite(reserve_percent)): + return False + expected_source = DISABLED_OPTION + expected_target = float(math.ceil(float(reserve_percent))) + for entity in sources: + observed = base.get_state_wrapper(entity_id=entity, raw=False, refresh=False) + if not inverter.service_sequence_state_matches(observed, expected_source): + return False + for entity in targets: + observed = base.get_state_wrapper(entity_id=entity, raw=False, refresh=False) + if not inverter.service_sequence_state_matches(observed, expected_target): + return False + return True diff --git a/apps/predbat/output.py b/apps/predbat/output.py index e802012f0..2127ec9f2 100644 --- a/apps/predbat/output.py +++ b/apps/predbat/output.py @@ -132,6 +132,12 @@ def publish_car_plan(self): """ Publish the car charging plan """ + # Fetch calls this before execution. Explicit joint apply must not leak + # the independently optimised legacy EV plan to the existing HA executor. + from joint_site_apply import publish_joint_ev + + if publish_joint_ev(self): + return postfix = "" for car_n in range(self.num_cars): # A list of its own per car - shared, every car after the first also carried the windows of @@ -1648,13 +1654,13 @@ def import_rate_color(rate): # charging in the same step, not only when no car is charging car_charging_cancelled = self.car_charge_slot_kwh_cancelled(minute_start, minute_end) if car_charging_kwh > 0.0: - car_charging_str = str(car_charging_kwh) + car_charging_str = "{:.2f}".format(car_charging_kwh) if car_charging_cancelled > 0.0: - car_charging_str += " +" + str(car_charging_cancelled) + "?" + car_charging_str += " +{:.2f}?".format(car_charging_cancelled) car_color = "FFFF00" car_rate = self.car_charge_slot_rate(minute_start, minute_end) elif car_charging_cancelled > 0.0: - car_charging_str = str(car_charging_cancelled) + "?" + car_charging_str = "{:.2f}?".format(car_charging_cancelled) car_color = "#FFFFCC" else: car_charging_str = "⚊" @@ -1901,7 +1907,7 @@ def import_rate_color(rate): if plan_debug and self.load_forecast: html += "{}".format(dp2(xload_total)) if self.num_cars > 0: - html += "{}".format(dp2(car_total)) + html += "{:.2f}".format(car_total) if self.iboost_enable: iboost_amount_str = "⚊" iboost_amount_end = self.predict_iboost_best.get(minute_relative_slot_end, 0) diff --git a/apps/predbat/plan.py b/apps/predbat/plan.py index 57332b6ef..be22a74c4 100644 --- a/apps/predbat/plan.py +++ b/apps/predbat/plan.py @@ -358,7 +358,12 @@ def car_charging_slots_model(self): """ now_slots = self.car_charging_now_slots # Always a fresh list per car, so a caller that edits the model can never edit the published plan - return [(now_slots[car_n] if car_n < len(now_slots) else []) + list(slots) for car_n, slots in enumerate(self.car_charging_slots)] + model = [(now_slots[car_n] if car_n < len(now_slots) else []) + list(slots) for car_n, slots in enumerate(self.car_charging_slots)] + from joint_site_arrival import model_arrival_slots + + if model and not model[0]: + model[0] = model_arrival_slots(self) + return model def car_charging_threshold_kw(self): """ @@ -2405,6 +2410,12 @@ def calculate_plan(self, recompute=True, debug_mode=False, publish=True): self.log(line) self.publish_html_plan(pv_forecast_minute_step, pv_forecast_minute10_step, load_minutes_step, load_minutes_step10, self.end_record) + # The personal-fork joint optimiser uses these forecasts in isolation. It + # never replaces the executable battery/EV plan or its Prediction object. + from joint_site_native import evaluate_joint_site + + evaluate_joint_site(self, publish=publish) + # Record planning duration for SLO metrics self.plan_last_duration_seconds = time.time() - plan_start_time metrics().planning_duration_seconds.observe(self.plan_last_duration_seconds) diff --git a/apps/predbat/predbat.py b/apps/predbat/predbat.py index 6981babd2..449ed0ae6 100644 --- a/apps/predbat/predbat.py +++ b/apps/predbat/predbat.py @@ -1193,7 +1193,9 @@ def update_pred(self, scheduled=True): self.log("Will not recompute the plan, it is {} minutes old and max age is {} minutes".format(dp1(plan_age_minutes), self.calculate_plan_every)) # Notify listeners that plan has been executed (consumed by gateway, plugins, etc.) - if self.plugin_system: + from joint_site_apply import native_plan_hook_allowed + + if self.plugin_system and native_plan_hook_allowed(self): self.plugin_system.call_hooks( "on_plan_executed", charge_windows=self.charge_window_best, @@ -2180,6 +2182,9 @@ def update_time_loop(self, cb_args): self.check_entity_refresh() self.validate_config_check_retry() if not self.prediction_started: + from joint_site_apply import refresh_joint_ev_lease + + refresh_joint_ev_lease(self) # A car's charging slots cancelled or resumed by dynamic load - replan now, not in 5 minutes self.dynamic_load_car_poll() # A car starting or stopping charging - apply or release "Hold for car" now, not in 5 minutes diff --git a/apps/predbat/prediction.py b/apps/predbat/prediction.py index d5310117b..2a8cf3630 100644 --- a/apps/predbat/prediction.py +++ b/apps/predbat/prediction.py @@ -18,7 +18,8 @@ """ from datetime import timedelta -from const import PREDICT_STEP, PV_SCENARIO_PV10, PV_SCENARIO_PV90, RUN_EVERY, TIME_FORMAT, EXPORT_LIMIT_FREEZE, EXPORT_LIMIT_IDLE, EXPORT_MODE_TARGET, EXPORT_MODE_FREEZE, EXPORT_MODE_IDLE +import math +from const import PREDICT_STEP, PV_SCENARIO_PV10, PV_SCENARIO_PV90, RUN_EVERY, TIME_FORMAT, EXPORT_LIMIT_FREEZE, EXPORT_LIMIT_IDLE, EXPORT_MODE_TARGET, EXPORT_MODE_FREEZE, EXPORT_MODE_IDLE, MINUTE_WATT from utils import ( remove_intersecting_windows, @@ -69,6 +70,65 @@ def get_total_inverted(battery_draw, pv_dc, pv_ac, inverter_loss, inverter_hybri return total_inverted +def _read_joint_grid_charge_support_w(base): + """Validate the internal, default-off shadow flag before constructing any kernel context.""" + raw = getattr(base, "joint_grid_charge_support_w", 0.0) + if isinstance(raw, bool) or not isinstance(raw, (int, float)) or not math.isfinite(raw) or raw < 0: + raise ValueError("Invalid joint grid-charge support setpoint") + return float(raw) + + +def compute_joint_grid_charge_support( + support_w, + load_kwh, + pv_now_kwh, + inverter_hybrid, + inverter_loss, + charge_rate_step, + charge_headroom_dc, + discharge_rate_now_curve_step, + battery_to_min, + inverter_limit, + step, +): + """Return signed DC battery flow, PV allocation and an intra-step import envelope. + + Energy arguments are kWh per step. PV serves load first; only surplus hybrid + PV charges directly on DC. Grid headroom is AC and incurs conversion loss. + A charge request may instead supply a deficit, bounded by existing discharge + rate, reserve energy and remaining inverter throughput. Average energy cannot + hide a reserve reached mid-step: the returned peak includes the unsupported tail. + """ + cap_energy = support_w * step / MINUTE_WATT + if inverter_hybrid: + pv_for_load_dc = min(pv_now_kwh, load_kwh / inverter_loss, inverter_limit) + pv_surplus_dc = max(pv_now_kwh - pv_for_load_dc, 0) + grid_load = max(load_kwh - pv_for_load_dc * inverter_loss, 0) + else: + pv_surplus_dc = 0 + grid_load = load_kwh - pv_now_kwh + deficit_dc = max(grid_load - cap_energy, 0) / inverter_loss + peak_energy = 0.0 + if deficit_dc > 0: + conversion_room = max(inverter_limit - (min(pv_now_kwh, inverter_limit) if inverter_hybrid else 0), 0) + full_discharge = min(deficit_dc, discharge_rate_now_curve_step, conversion_room) + battery_draw = min(full_discharge, battery_to_min) + pv_dc = 0.0 + if battery_to_min < full_discharge - 1e-12: + peak_energy = max(grid_load, 0) + else: + full_charge = min(charge_rate_step, pv_surplus_dc + max(cap_energy - grid_load, 0) * inverter_loss) + charge = min(full_charge, max(charge_headroom_dc, 0)) + battery_draw = -charge + pv_dc = min(charge, pv_surplus_dc) + if charge > 0 and charge < full_charge - 1e-12: + # Charging finishes before the step ends: retain its initial power, + # not just the smaller time-averaged energy shown in the normal trace. + peak_energy = max(grid_load + max(full_charge - pv_surplus_dc, 0) / inverter_loss, 0) + pv_ac = (pv_now_kwh - pv_dc) * (inverter_loss if inverter_hybrid else 1) + return battery_draw, pv_dc, pv_ac, peak_energy * MINUTE_WATT / step + + class Prediction(PredictionBatch): """ Class to hold prediction input and output data and the run function @@ -99,6 +159,8 @@ def __init__( car_charging_slots, when given, replaces base.car_charging_slots - the live plan passes car_charging_slots_model() so a car charging now outside its plan is modelled too. """ + self.joint_grid_charge_support_w = _read_joint_grid_charge_support_w(base) + self.joint_grid_support_import = None if base: self.minutes_now = base.minutes_now self.log = base.log @@ -217,7 +279,7 @@ def __init__( self.charge_scaling10 = base.charge_scaling10 # C++ prediction kernel context (0 = kernel unavailable, Python engine is used) - self.prediction_kernel_enable = getattr(base, "prediction_kernel_enable", False) + self.prediction_kernel_enable = getattr(base, "prediction_kernel_enable", False) and not self.joint_grid_charge_support_w self.kernel_handle = 0 if self.prediction_kernel_enable: self.kernel_handle = create_kernel_context(self, static_cache=kernel_static_cache) @@ -578,6 +640,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi predict_pv_power = {} predict_state = {} predict_grid_power = {} + self.joint_grid_support_import = {} if self.joint_grid_charge_support_w else None predict_load_power = {} predict_iboost = {} predict_carbon_g = {} @@ -731,6 +794,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi # Simulate each forward minute minute = 0 while minute < self.forecast_minutes: + support_peak_w = 0.0 # Minute yesterday can wrap if days_previous is only 1 minute_absolute = minute + self.minutes_now prev_soc = soc @@ -1153,6 +1217,23 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi pv_in_period = pv_compare / step * charge_time_remains potential_import = min((charge_rate_now_curve * charge_time_remains) - pv_in_period, (charge_limit_n - soc)) metric_keep += max(potential_import * import_rate, 0) + + # Explicit shadow-only physics; no flag is placed on the live base. + if self.joint_grid_charge_support_w: + battery_draw, pv_dc, pv_ac, support_peak_w = compute_joint_grid_charge_support( + self.joint_grid_charge_support_w, + load_yesterday, + pv_now, + inverter_hybrid, + inverter_loss, + charge_rate_now_curve_step, + max(min(charge_limit_n, soc_max) - soc, 0) / battery_loss, + discharge_rate_now_curve_step, + battery_to_min, + inverter_limit, + step, + ) + battery_state = "f+" if battery_draw < 0 else "f/" if battery_draw > 0 else "f~" else: # ECO Mode. # @@ -1361,6 +1442,14 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi # The car in a dispatch that has gone away is met by the grid (or PV surplus), never the battery diff += car_gone_kwh + if self.joint_grid_support_import is not None: + # Raw whole-horizon evidence, including the intra-step boundary peak. + # Keep the cost/energy trace unchanged rather than replacing it with + # worst-case power. The optional import diagnostics remain independent. + average_w = max(diff + car_load_energy_bypass, 0) * MINUTE_WATT / step + peak_w = max(average_w, support_peak_w + (car_gone_kwh + car_load_energy_bypass) * MINUTE_WATT / step) + self.joint_grid_support_import[minute] = (average_w, peak_w) + # Metric keep - pretend the battery is empty and you have to import instead of using the battery if best_soc_keep > 0 and soc <= best_soc_keep: metric_keep += (best_soc_keep - soc) * import_rate * keep_minute_scaling * step / 60.0 diff --git a/apps/predbat/storage.py b/apps/predbat/storage.py index 70944445b..8ef53a9a4 100644 --- a/apps/predbat/storage.py +++ b/apps/predbat/storage.py @@ -366,6 +366,17 @@ async def load(self, module, filename): self.log("Storage: Warn: Failed to deserialise {}: {}".format(data_path, e)) return None + async def load_checked(self, module, filename): + """Distinguish first use from an unreadable, expired or orphaned durable record. + + Ordinary caches retain their existing permissive ``load`` behaviour. + Authority records must not silently turn corruption into a new session. + """ + value = await self.load(module, filename) + if value is None and (os.path.exists(self._meta_path(module, filename)) or any(os.path.exists(self._data_path(module, filename, fmt)) for fmt in STORAGE_FORMATS)): + raise ValueError("Stored record exists but cannot be loaded") + return value + async def age(self, module, filename): """Return the age of stored data in minutes since it was created. @@ -546,6 +557,10 @@ async def load(self, module, filename): """ return await self.backend.load(module, filename) + async def load_checked(self, module, filename): + """Load a durable record without treating existing unreadable data as first use.""" + return await self.backend.load_checked(module, filename) + async def age(self, module, filename): """Return the age of stored data in minutes since it was created. diff --git a/apps/predbat/tests/test_joint_site_actual.py b/apps/predbat/tests/test_joint_site_actual.py new file mode 100644 index 000000000..4919b643a --- /dev/null +++ b/apps/predbat/tests/test_joint_site_actual.py @@ -0,0 +1,434 @@ +"""Real-template acceptance for bounded charge ceilings and confirmed unchanged EV sessions.""" + +from copy import deepcopy +from datetime import timedelta +from unittest.mock import patch + +from joint_site_apply import execute_joint_site, publish_joint_ev, refresh_joint_ev_lease +from joint_site_native import _options, _continuous_grid_support_shadow, evaluate_joint_site +from joint_site_observation import confirmed_session_observation +from tests.test_joint_site_combined import _template_inputs +from tests.test_joint_site_native import _set_entity as _cached_entity + + +def _set_entity(base, entity, state, attributes=None, **kwargs): + """Keep both native mock attribute lookups and raw HA payload attributes available.""" + _cached_entity(base, entity, state, attributes={**(attributes or {}), "attributes": attributes or {}}, **kwargs) + + +def _actual_inputs(base): + """Build real native controls with an explicit confirmation and old but valid retained states.""" + inverter, writes, sources, targets = _template_inputs(base, pv=0) + base.soc_kw = base.soc_max * 0.26 + inverter.soc_kw, inverter.soc_percent = base.soc_kw, 26 + base.all_active_keep = {base.minutes_now + minute: 25 for minute in range(5)} + settings = base.args["joint_site"] + settings["session"]["available_from"] = (base.now_utc - timedelta(hours=1)).isoformat() + settings.update( + mode="apply", + apply_verified=True, + ev_executor_lease_verified=True, + grid_charge_support={"enabled_entity": "switch.test_support", "limit_entity": "number.test_support"}, + confirmed_session={ + "confirmed_at": (base.now_utc - timedelta(minutes=20)).isoformat(), + "departure": settings["session"]["departure"], + "soc_kwh": 0, + "energy_entity": "sensor.test_ev_energy", + "energy_kwh": 100, + "power_entity": "sensor.test_ev_power", + }, + ) + _set_entity(base, "switch.test_support", "on") + _set_entity(base, "number.test_support", "6000") + for key, value in (("soc_entity", 0), ("connection_entity", "connected")): + _set_entity(base, settings[key], value, last_updated=base.now_utc - timedelta(hours=12), attributes={"retrievalstatus": "VALID"}) + _set_entity(base, "sensor.test_ev_energy", "100", last_updated=base.now_utc - timedelta(hours=12), attributes={"unit_of_measurement": "kWh"}) + _set_entity(base, "sensor.test_ev_power", "0", attributes={"unit_of_measurement": "kW"}) + return inverter, writes, sources, targets + + +def test_confirmed_unchanged_soc_and_meter(base): + """Unchanged valid values retain their real timestamps and delivery advances only from the meter.""" + _actual_inputs(base) + settings = base.args["joint_site"] + original = deepcopy(base.ha_interface.dummy_items) + value = confirmed_session_observation(base, settings, base.now_utc) + assert value["ev_soc"] == 0 and value["ev_soc_is_estimate"] and value["soc_observation_age_minutes"] == 720 + assert value["ev_soc_source"] == "operator_confirmed_metered_session" and base.ha_interface.dummy_items == original + _set_entity(base, "sensor.test_ev_energy", "101", attributes={"unit_of_measurement": "kWh"}) + value = confirmed_session_observation(base, settings, base.now_utc) + assert value["ev_metered_ac_kwh"] == 1 and abs(value["ev_soc"] - base.car_charging_loss) < 1e-10 + _set_entity(base, settings["soc_entity"], 1.5, attributes={"retrievalstatus": "VALID"}) + assert confirmed_session_observation(base, settings, base.now_utc)["ev_soc"] == 1.5 + assert _options(base)["entity_blockers"] == [] + + +def test_confirmation_fails_closed_without_fabricating_clock(base): + """A reset meter, invalid field or changed deadline cannot silently reuse the confirmation.""" + for fault in ("meter_reset", "missing_meter", "bad_soc", "invalid_status", "future", "renewed_deadline", "energy_stalled"): + _actual_inputs(base) + settings = base.args["joint_site"] + if fault == "meter_reset": + _set_entity(base, "sensor.test_ev_energy", "99", attributes={"unit_of_measurement": "kWh"}) + elif fault == "missing_meter": + base.ha_interface.dummy_items.pop("sensor.test_ev_energy") + elif fault == "bad_soc": + _set_entity(base, settings["soc_entity"], "unavailable", attributes={"retrievalstatus": "VALID"}) + elif fault == "invalid_status": + _set_entity(base, settings["soc_entity"], 0, attributes={"retrievalstatus": "INVALID"}) + elif fault == "future": + settings["confirmed_session"]["confirmed_at"] = (base.now_utc + timedelta(seconds=1)).isoformat() + elif fault == "renewed_deadline": + settings["session"]["departure"] = (base.now_utc + timedelta(hours=2)).isoformat() + else: + _set_entity(base, settings["relay_entity"], "on", last_changed=base.now_utc - timedelta(minutes=11)) + _set_entity(base, "sensor.test_ev_power", "6.77", attributes={"unit_of_measurement": "kW"}) + try: + confirmed_session_observation(base, settings, base.now_utc) + except ValueError: + pass + else: + raise AssertionError("Invalid confirmation accepted: " + fault) + + +def test_actual_charge_ceiling_matches_replay_and_lease(base): + """The real executor writes the replayed integer ceiling, never replaces it with 100 percent.""" + inverter, writes, sources, targets = _actual_inputs(base) + report = evaluate_joint_site(base, publish=False) + assert report["status"] == "candidate", report + assert all(check["passed"] for check in report["native_checks"].values()) + assert not writes and not report["execution_enabled"] + command = deepcopy(base.joint_site_pending["commands"][0]) + assert command["ev_on"] and 10 < command["target_soc"] < 100, command + assert execute_joint_site(base)[0] == "Charging", base.joint_site_report + assert all(base.ha_interface.get_state(entity) == "Grid" for entity in sources) + assert all(float(base.ha_interface.get_state(entity)) == command["target_soc"] for entity in targets) + assert base.joint_site_report["status"] == "applied" and base.joint_site_report["execution_enabled"] + assert not any(service.startswith("switch") for service, _ in writes) + signal = base.dashboard_values["binary_sensor." + base.prefix + "_car_charging_slot"] + assert signal["state"] == "on" and signal["attributes"]["source"] == "joint_site_apply" + count = len(writes) + with patch("joint_site_native._observation_clock", return_value=base.now_utc + timedelta(seconds=301)): + publish_joint_ev(base) + assert base.joint_site_execution is None and len(writes) == count + assert base.dashboard_values["binary_sensor." + base.prefix + "_car_charging_slot"]["state"] == "off" + + +def test_actual_independent_gates_and_cap_drift(base): + """Operator evidence never overrides read-only, permission, reserve or a changed support setting.""" + for fault in ("read_only", "permission", "support_disabled", "target", "commissioning", "lease", "connection"): + _, writes, _, _ = _actual_inputs(base) + report = evaluate_joint_site(base, publish=False) + assert report["status"] == "candidate" + settings = base.args["joint_site"] + if fault == "read_only": + base.set_read_only = True + elif fault == "permission": + _set_entity(base, settings["permission_entity"], "off") + elif fault == "support_disabled": + _set_entity(base, "switch.test_support", "off") + elif fault == "target": + base.joint_site_pending["commands"][0]["target_soc"] = 101 + elif fault == "commissioning": + settings["apply_verified"] = False + elif fault == "lease": + settings["ev_executor_lease_verified"] = False + else: + _set_entity(base, settings["connection_entity"], "disconnected", attributes={"retrievalstatus": "VALID"}) + assert execute_joint_site(base)[0] == "Joint blocked" and not writes + assert not base.joint_site_execution + + +def test_confirmed_session_progress_and_failures(base): + """Repeated unchanged SOC cannot freeze progress; invalid meters and clocks fail closed.""" + _actual_inputs(base) + settings = base.args["joint_site"] + _set_entity(base, "sensor.test_ev_energy", "101", attributes={"unit_of_measurement": "kWh"}) + _set_entity(base, settings["soc_entity"], 1.5, attributes={"retrievalstatus": "VALID"}) + assert confirmed_session_observation(base, settings, base.now_utc)["ev_soc"] == 1.5 + later = base.now_utc + timedelta(minutes=5) + _set_entity(base, "sensor.test_ev_energy", "102", last_updated=later, attributes={"unit_of_measurement": "kWh"}) + first = confirmed_session_observation(base, settings, later) + assert first["ev_soc"] == 2 * base.car_charging_loss and first["ev_soc_is_estimate"] + assert confirmed_session_observation(base, deepcopy(settings), later) == first + for fault in ("zero_power_stall", "future_transition", "nan", "bool", "huge_power", "fast_counter", "attributes", "before_arrival", "soc_contradiction"): + _actual_inputs(base) + settings = base.args["joint_site"] + if fault == "zero_power_stall": + _set_entity(base, settings["relay_entity"], "on", last_changed=base.now_utc - timedelta(minutes=11)) + elif fault == "future_transition": + _set_entity(base, settings["relay_entity"], "off", last_changed=base.now_utc + timedelta(seconds=1)) + elif fault in ("nan", "bool"): + _set_entity(base, "sensor.test_ev_energy", float("nan") if fault == "nan" else True, attributes={"unit_of_measurement": "kWh"}) + elif fault == "huge_power": + _set_entity(base, "sensor.test_ev_power", "30", attributes={"unit_of_measurement": "kW"}) + elif fault == "fast_counter": + _set_entity(base, "sensor.test_ev_energy", "105", attributes={"unit_of_measurement": "kWh"}) + elif fault == "attributes": + base.ha_interface.dummy_items["sensor.test_ev_energy"]["attributes"] = None + elif fault == "before_arrival": + settings["confirmed_session"]["confirmed_at"] = (base.now_utc - timedelta(hours=2)).isoformat() + else: + _set_entity(base, settings["soc_entity"], 9, attributes={"retrievalstatus": "VALID"}) + try: + confirmed_session_observation(base, settings, base.now_utc) + except ValueError: + pass + else: + raise AssertionError("Invalid confirmation accepted: " + fault) + + +def test_confirmed_session_closes_without_renewal(base): + """After departure, missing EV telemetry cannot block household planning or renew EV charging.""" + _, writes, _, _ = _actual_inputs(base) + settings = base.args["joint_site"] + settings["session"]["departure"] = settings["confirmed_session"]["departure"] = (base.now_utc - timedelta(minutes=1)).isoformat() + for name in ("soc_entity", "connection_entity"): + base.ha_interface.dummy_items.pop(settings[name]) + base.ha_interface.dummy_items.pop("sensor.test_ev_energy") + options = _options(base) + assert options["house_only"] and options["confirmed_observation"]["session_closed"] + report = evaluate_joint_site(base, publish=False) + assert report["status"] == "candidate" and report["observed_ev_target_met"] is None, report + assert not report["proposed_ev_windows"] and report["target_met_by_departure"] is None + assert not any(command["ev_on"] for command in base.joint_site_pending["commands"]) + assert execute_joint_site(base)[0] != "Joint blocked", base.joint_site_report + assert not any(service.startswith("switch") for service, _ in writes) + + +def test_actual_partial_dispatch_retry_after_restart(base): + """Partial native writes revoke EV authority; restart-like state loss requires full revalidation.""" + for fail_phase in range(3): + inverter, writes, sources, targets = _actual_inputs(base) + assert evaluate_joint_site(base, publish=False)["status"] == "candidate" + target = base.joint_site_pending["commands"][0]["target_soc"] + for entity in sources: + _set_entity(base, entity, "Grid", attributes={"options": ["Disabled", "Grid", "Gen", "Grid+Gen"]}) + original = base.call_service_wrapper + calls = [] + + def failing(service, **data): + """Interrupt one actual service phase while retaining partial cached changes.""" + calls.append(service) + return False if len(calls) == fail_phase + 1 else original(service, **data) + + base.call_service_wrapper = failing + assert execute_joint_site(base)[0] == "Joint blocked" + assert not base.joint_site_execution + assert base.dashboard_values["binary_sensor." + base.prefix + "_car_charging_slot"]["state"] == "off" + base.call_service_wrapper = original + base.had_errors = False + base.joint_site_pending = None + assert execute_joint_site(base)[0] == "Joint blocked" + base.had_errors = False + assert evaluate_joint_site(base, publish=False)["status"] == "candidate" + assert execute_joint_site(base)[0] == "Charging", base.joint_site_report + assert all(float(base.ha_interface.get_state(entity)) == target for entity in targets) + assert all(base.ha_interface.get_state(entity) == "Grid" for entity in sources) + assert not any(service.startswith("switch") for service, _ in writes) + + +def test_actual_lease_renewal_is_bounded_and_does_not_dispatch(base): + """Cached renewal bridges calculation jitter but never extends an old plan indefinitely.""" + _, writes, _, _ = _actual_inputs(base) + base.all_active_keep = {} + base.soc_kw = base.soc_max * 0.8 + base.inverters[0].soc_kw, base.inverters[0].soc_percent = base.soc_kw, 80 + assert evaluate_joint_site(base, publish=False)["status"] == "candidate" + assert execute_joint_site(base)[0] != "Joint blocked", base.joint_site_report + initial = deepcopy(base.joint_site_execution) + count = len(writes) + # Keep a fully accepted current command across consecutive equal five-minute steps. + assert initial["command"]["end"] > base.minutes_now + 10 + for seconds in (240, 480): + with patch("joint_site_native._observation_clock", return_value=base.now_utc + timedelta(seconds=seconds)): + refresh_joint_ev_lease(base) + receipt = base.joint_site_execution + assert receipt and receipt["plan_generated_at"] == initial["plan_generated_at"] + assert len(writes) == count + from joint_site_preview import build_joint_plan_view + + assert build_joint_plan_view({}, base.joint_site_report, now=base.now_utc + timedelta(seconds=seconds))["joint_execution"] is True + with patch("joint_site_native._observation_clock", return_value=base.now_utc + timedelta(seconds=600)): + refresh_joint_ev_lease(base) + assert base.joint_site_execution is None and len(writes) == count + assert base.dashboard_values["binary_sensor." + base.prefix + "_car_charging_slot"]["state"] == "off" + + +def test_explicit_session_authority_is_not_hardware_attestation(base): + """Only the explicit confirmed shared-support scope may run with an unverified attestation.""" + for confirmed in (False, True): + _, writes, _, _ = _actual_inputs(base) + settings = base.args["joint_site"] + settings.update(apply_verified=False, execution_authorised=True) + if not confirmed: + settings.pop("confirmed_session") + assert execute_joint_site(base)[0] == "Joint blocked" and not writes + else: + assert evaluate_joint_site(base, publish=False)["status"] == "candidate" + assert execute_joint_site(base)[0] == "Charging", base.joint_site_report + assert settings["apply_verified"] is False and base.joint_site_report["hardware_confirmed"] is False + + +def test_dispatch_replays_soc_drop_and_binds_exact_commands(base): + """A later SOC observation requires real paired replay; forged in-range targets fail closed.""" + for case in ("small_drop", "unsafe_drop", "forged_target"): + _, writes, _, _ = _actual_inputs(base) + assert evaluate_joint_site(base, publish=False)["status"] == "candidate" + expected = deepcopy(base.joint_site_pending["commands"]) + if case == "forged_target": + base.joint_site_pending["commands"][0]["target_soc"] += 1 + else: + base.soc_kw -= 0.01 if case == "small_drop" else base.soc_max * 0.08 + status = execute_joint_site(base) + if case == "small_drop": + assert status[0] == "Charging", base.joint_site_report + assert base.joint_site_pending["commands"] == expected + assert base.joint_site_report["dispatch_soc_revalidated_kwh"] == base.soc_kw + assert all(check["passed"] for check in base.joint_site_report["native_checks"].values()) + else: + assert status[0] == "Joint blocked" and not writes + + +def test_operator_stop_acknowledgement_is_one_observation_only(base): + """An acknowledged manual-stop error never permits new errors or weakens independent guards.""" + for case in ("accepted", "absent", "later_error", "other_error", "future", "wrong_departure", "read_only", "permission", "meter_reset"): + _, writes, _, _ = _actual_inputs(base) + settings = base.args["joint_site"] + observed = base.now_utc - timedelta(minutes=10) + acknowledgement = {"state": "4", "observed_at": observed.isoformat(), "acknowledged_at": base.now_utc.isoformat(), "departure": settings["session"]["departure"]} + settings["connection_error_acknowledgement"] = acknowledgement + _set_entity(base, settings["connection_entity"], "4", last_updated=observed, attributes={"retrievalstatus": "VALID"}) + if case == "absent": + settings.pop("connection_error_acknowledgement") + elif case == "later_error": + _set_entity(base, settings["connection_entity"], "4", last_updated=base.now_utc, attributes={"retrievalstatus": "VALID"}) + elif case == "other_error": + _set_entity(base, settings["connection_entity"], "16", last_updated=observed, attributes={"retrievalstatus": "VALID"}) + elif case == "future": + acknowledgement["acknowledged_at"] = (base.now_utc + timedelta(seconds=1)).isoformat() + elif case == "wrong_departure": + acknowledgement["departure"] = (base.now_utc + timedelta(days=1)).isoformat() + assert evaluate_joint_site(base, publish=False)["status"] == "candidate" + if case == "read_only": + base.set_read_only = True + elif case == "permission": + _set_entity(base, settings["permission_entity"], "off") + elif case == "meter_reset": + _set_entity(base, "sensor.test_ev_energy", "99", attributes={"unit_of_measurement": "kWh"}) + status = execute_joint_site(base) + if case == "accepted": + assert status[0] == "Charging", base.joint_site_report + assert base.joint_site_execution["connection_error_acknowledged"] is True + assert "4" not in settings["connection_entity_states"] + else: + assert status[0] == "Joint blocked" and not writes, (case, base.joint_site_report) + + +def test_native_readonly_switch_transition_requires_observed_dispatch(base): + """Fulfil only the native read-only reset request using the verified joint rows, not a flag bypass.""" + for case in ("accepted", "failed_write", "other_reset", "relay_on"): + _, writes, _, _ = _actual_inputs(base) + base.expose_config("set_read_only", True, event=True) + base.expose_config("set_read_only", False, event=True) + assert base.inverter_needs_reset is True and base.inverter_needs_reset_force == "set_read_only" + base.set_read_only = False + assert evaluate_joint_site(base, publish=False)["status"] == "candidate" + if case == "failed_write": + base.call_service_wrapper = lambda *a, **k: False + elif case == "other_reset": + base.inverter_needs_reset_force = "mode" + elif case == "relay_on": + _set_entity(base, base.args["joint_site"]["relay_entity"], "on") + status = execute_joint_site(base) + if case == "accepted": + assert status[0] == "Charging", base.joint_site_report + assert base.inverter_needs_reset is False and base.inverter_needs_reset_force == "" + assert base.joint_site_execution["read_only_transition_observed"] is True + assert all(service in ("number/set_value", "select/select_option") and "_program_" in data["entity_id"] for service, data in writes) + else: + assert status[0] == "Joint blocked" and base.inverter_needs_reset is True + assert base.joint_site_execution is None + + +def test_asap_prefers_earlier_paired_block_without_changing_session(base): + """ASAP takes the next cheaper band before the cheapest later band without changing consent.""" + _actual_inputs(base) + settings = base.args["joint_site"] + settings["session"]["departure"] = settings["confirmed_session"]["departure"] = (base.now_utc + timedelta(minutes=60)).isoformat() + base.soc_kw = base.soc_max * 0.8 + base.inverters[0].soc_kw, base.inverters[0].soc_percent = base.soc_kw, 80 + base.metric_battery_value_scaling = 0 + base.rate_import.update({base.minutes_now + minute: 50 if minute < 15 else 10 if minute < 30 else 1 for minute in range(60)}) + session = deepcopy(settings["session"]) + confirmation = deepcopy(settings["confirmed_session"]) + ordinary = evaluate_joint_site(base, publish=False) + assert ordinary["status"] == "candidate" and ordinary["proposed_ev_windows"][0]["start"] > base.minutes_now + settings["ev_charging_priority"] = "asap" + report = evaluate_joint_site(base, publish=False) + assert report["status"] == "candidate", report + assert report["ev_charging_priority"] == "asap" and report["proposed_ev_windows"][0]["start"] == base.minutes_now + 15 + assert report["proposed_ev_windows"][0]["start"] < ordinary["proposed_ev_windows"][0]["start"] + assert report["priority_selection_reason"] == "next_cheaper_band" + assert len(report["continuous_block_attempts"]) == 1 and all(check["passed"] for check in report["native_checks"].values()) + assert all(left["end"] == right["start"] for left, right in zip(report["proposed_ev_windows"], report["proposed_ev_windows"][1:])) + assert settings["session"] == session and settings["confirmed_session"] == confirmation + assert not base.joint_site_pending["commands"][0]["ev_on"] + assert execute_joint_site(base)[0] != "Joint blocked", base.joint_site_report + assert base.dashboard_values["binary_sensor." + base.prefix + "_car_charging_slot"]["state"] == "off" + # A later priority edit invalidates the old receipt instead of silently reusing it. + settings["ev_charging_priority"] = "cost" + publish_joint_ev(base) + assert base.joint_site_execution is None + + +def test_asap_respects_cooldown_and_both_forecasts(base): + """ASAP cannot bypass cooldown, a failed replay, scope validation or the twelve-search bound.""" + _actual_inputs(base) + settings = base.args["joint_site"] + settings.update(ev_charging_priority="asap") + settings["session"]["departure"] = settings["confirmed_session"]["departure"] = (base.now_utc + timedelta(minutes=60)).isoformat() + _set_entity(base, settings["relay_entity"], "off", last_changed=base.now_utc - timedelta(minutes=5)) + options = _options(base) + chosen, attempts = _continuous_grid_support_shadow(base, options) + assert chosen and attempts[0]["start_offset"] == 5 and chosen[2][0][0]["start"] >= base.minutes_now + 5 + with patch("joint_site_native._replay_candidate", return_value=(chosen[2], {"nominal": {"passed": True}, "pv10": {"passed": False}})): + rejected, failed = _continuous_grid_support_shadow(base, options) + assert rejected is None and 1 <= len(failed) <= 12 and all(not attempt["native_passed"] for attempt in failed) + assert [a["start_offset"] for a in failed] == sorted(a["start_offset"] for a in failed) + assert _continuous_grid_support_shadow(base, {**options, "house_only": True}) == (None, []) + for value in (None, True, "ASAP", "fastest"): + settings["ev_charging_priority"] = value + assert evaluate_joint_site(base, publish=False)["status"] == "invalid_input" + settings["ev_charging_priority"] = "asap" + settings.pop("grid_charge_support") + assert evaluate_joint_site(base, publish=False)["status"] == "invalid_input" + + +def run_joint_site_actual_tests(base): + """Run isolated actual-plan tests with a deterministic observation clock.""" + from tests.test_joint_site_template import shared_template_available, check_unavailable_template + + if not shared_template_available(): + check_unavailable_template(base) + return False + + with patch("joint_site_native._observation_clock", side_effect=lambda: base.now_utc): + for test in ( + test_confirmed_unchanged_soc_and_meter, + test_confirmation_fails_closed_without_fabricating_clock, + test_actual_charge_ceiling_matches_replay_and_lease, + test_actual_independent_gates_and_cap_drift, + test_confirmed_session_progress_and_failures, + test_confirmed_session_closes_without_renewal, + test_actual_partial_dispatch_retry_after_restart, + test_actual_lease_renewal_is_bounded_and_does_not_dispatch, + test_explicit_session_authority_is_not_hardware_attestation, + test_dispatch_replays_soc_drop_and_binds_exact_commands, + test_operator_stop_acknowledgement_is_one_observation_only, + test_native_readonly_switch_transition_requires_observed_dispatch, + test_asap_prefers_earlier_paired_block_without_changing_session, + test_asap_respects_cooldown_and_both_forecasts, + ): + test(base) + return False diff --git a/apps/predbat/tests/test_joint_site_applied_view.py b/apps/predbat/tests/test_joint_site_applied_view.py new file mode 100644 index 000000000..dea9dba5b --- /dev/null +++ b/apps/predbat/tests/test_joint_site_applied_view.py @@ -0,0 +1,742 @@ +"""Truthful applied-plan presentation tests for ``joint_site_preview``. + +The parent executor writes ``status='applied'`` with +``requested_mode='apply'``, ``execution_enabled=True``, +``actuator_signals_modified=True``, ``execution_observed_at``, +``execution_valid_until`` and may also write ``status='apply_blocked'`` on +its own failures. The preview module must show the actual corrected plan +when paired rows + fresh receipt + coherent ``generated_at`` are all in +force, and clear rows + surface a bounded ``execution_blocked`` reason +otherwise. Shadow candidate behaviour (``joint_simulation=True``) is +preserved. +""" + +import asyncio +import json +import re +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +from joint_site_preview import ( + _ALLOWED_BLOCKED_REASONS, + _applied_display_allowed, + _applied_receipt_fresh, + _paired_preview, + build_joint_plan_view, + render_joint_site_preview, +) +from web import WebInterface + + +def _now(): + """Return the fixed clock used by deterministic presentation tests.""" + return datetime(2026, 10, 12, 12, 0, 0, tzinfo=timezone.utc) + + +def _wall_now(): + """Anchor web-route fixtures to the same real clock used by the renderer.""" + return datetime.now(timezone.utc) + + +def _good_row(scenario="nominal", start="2026-01-01T12:00:00+0000", end="2026-01-01T12:30:00+0000", minutes=30, **overrides): + """Build a complete synthetic replay row without borrowing native values.""" + row = { + "start": start, + "end": end, + "minutes": minutes, + "battery_modes": "demand", + "ev_on_minutes": 0, + "ev_reserved_kwh": 0.0, + "house_kwh": 0.5, + "pv_kwh": 0.2, + "battery_start_percent": 50.0, + "battery_end_percent": 51.0, + "battery_min_percent": 50.0, + "grid_peak_w": 1500.0, + "mode_minutes": {"demand": minutes}, + } + for key, value in overrides.items(): + row[key] = value + return row + + +def _good_report( + status="candidate", + requested_mode="shadow", + execution_enabled=False, + actuator_signals_modified=False, + execution_observed_at=None, + execution_valid_until=None, + generated_minutes_ago=5, + native_passed=True, + now=None, + execution_reason=None, + uses_stale=False, + preview=None, + blockers=None, + counterfactual_controls=False, + status_block=None, +): + """Construct a paired report with independently selectable evidence gates.""" + now = now if now is not None else _now() + report = { + "status": status, + "requested_mode": requested_mode, + "execution_enabled": execution_enabled, + "actuator_signals_modified": actuator_signals_modified, + "generated_at": (now - timedelta(minutes=generated_minutes_ago)).isoformat(), + "uses_stale_soc_assumption": uses_stale, + "preview": preview + if preview is not None + else { + "nominal": [_good_row(scenario="nominal", import_rate=5.0, export_rate=0.0, cost_change=0.12, total_cost=1.25)], + "pv10": [_good_row(scenario="pv10", import_rate=5.0, export_rate=0.0)], + }, + "native_checks": {"nominal": {"passed": native_passed}, "pv10": {"passed": native_passed}}, + } + if execution_observed_at is not None: + report["execution_observed_at"] = execution_observed_at + if execution_valid_until is not None: + report["execution_valid_until"] = execution_valid_until + if execution_reason is not None: + report["execution_reason"] = execution_reason + if blockers is not None: + report["execution_blockers"] = blockers + if counterfactual_controls: + report["counterfactual_controls"] = True + if status_block is not None: + report["status"] = status_block + return report + + +def _applied_report(**overrides): + """Build a fresh applied receipt anchored to the wall clock so it survives + ``html_api_plan_data``-style ``datetime.now`` paths. ``generated_at`` is + pinned to ``observed_at - 1s`` to mirror the parent's executor.""" + now = overrides.pop("now", _wall_now()) + interval = overrides.pop("interval_seconds", 60) + observed = overrides.pop("observed_at", now - timedelta(seconds=1)) + valid_until = overrides.pop("valid_until", observed + timedelta(seconds=interval)) + base = { + "status": "applied", + "requested_mode": "apply", + "execution_enabled": True, + "actuator_signals_modified": True, + "execution_observed_at": observed.isoformat(), + "execution_valid_until": valid_until.isoformat(), + } + base.update(overrides) + base.setdefault("generated_minutes_ago", max(0.0, (now - observed).total_seconds() / 60.0)) + return _good_report(now=now, **base) + + +def _native_plan(): + """Provide native display metadata without a candidate trajectory.""" + return { + "timestamp": "2026-01-01T12:00:00+0000", + "time": "2026-01-01T00:00:00+0000", + "currency_symbols": ["£", "p"], + "num_cars": 0, + "iboost_enable": False, + "carbon_enable": False, + "import_cost_threshold": 10, + "export_cost_threshold": 0, + "rows": [], + } + + +def _native_diag(state, peak_w=None, limit_w=None, minutes_over=None): + """Represent the independent native import diagnostic.""" + attrs = {} + if peak_w is not None: + attrs["peak_w"] = peak_w + if limit_w is not None: + attrs["limit_w"] = limit_w + if minutes_over is not None: + attrs["minutes_over"] = minutes_over + return {"state": state, "attributes": attrs} + + +def _build_page(report, native_plan=None, diag=None): + """Create an in-memory web page backed only by the supplied reports.""" + native_plan = native_plan if native_plan is not None else _native_plan() + diag = diag if diag is not None else _native_diag("ok", peak_w=5400, limit_w=6000) + holder = SimpleNamespace(joint_site_report=report, manual_times=lambda *a, **k: [], manual_rates=lambda *a, **k: {}) + page = WebInterface.__new__(WebInterface) + page.base = holder + page.prefix = "predbat" + + def read(entity_id, attribute=None, default=None, raw=False): + """Serve isolated native metadata and diagnostics without Home Assistant.""" + if entity_id == "predbat.plan_html" and attribute == "raw": + return deepcopy(native_plan) + if entity_id == "predbat.best_import_limit" and raw: + return deepcopy(diag) + return default + + page.get_state_wrapper = read + page.get_header = lambda *a, **k: "" + return page + + +# ---------------------------------------------------------------------- +# Gap 1: valid applied view must set joint_simulation=False +# ---------------------------------------------------------------------- + + +def test_valid_applied_view_sets_joint_simulation_false(): + """Fresh observed execution is not labelled as a shadow simulation.""" + now = _wall_now() + view = build_joint_plan_view(_native_plan(), _applied_report(now=now), now=now) + assert view["joint_simulation"] is False, view + assert view["joint_execution"] is True, view + assert "execution_blocked" not in view, view + assert len(view["rows"]) == 1, view + + +def test_valid_applied_view_preserves_execution_dates(): + """Retain the original execution receipt dates without renewing them.""" + now = _wall_now() + report = _applied_report(now=now) + view = build_joint_plan_view(_native_plan(), report, now=now) + assert view["execution_observed_at"] == report["execution_observed_at"], view + assert view["execution_valid_until"] == report["execution_valid_until"], view + + +def test_shadow_candidate_keeps_joint_simulation_true(): + """Unexecuted shadow candidates must retain their simulation label.""" + now = _wall_now() + shadow = _good_report(now=now, status="candidate", requested_mode="shadow") + view = build_joint_plan_view(_native_plan(), shadow, now=now) + assert view["joint_simulation"] is True, view + assert "joint_execution" not in view, view + + +# ---------------------------------------------------------------------- +# Gap 2: render applied-fresh branch requires paired + checks + coherent generated +# ---------------------------------------------------------------------- + + +def test_render_fresh_applied_uses_active_plan_label(): + """Name the active plan without claiming the whole forecast was executed.""" + html = render_joint_site_preview( + _applied_report(now=_wall_now()), + _native_diag("ok", peak_w=5400, limit_w=6000), + now=_wall_now(), + ) + assert "Active plan" in html, html + assert "Applied plan (executed)" not in html, html + + +def test_render_fresh_applied_mainline_has_no_iso_dates(): + """Keep detailed timestamps under Show Debug rather than in the compact line.""" + html = render_joint_site_preview( + _applied_report(now=_wall_now()), + _native_diag("ok", peak_w=5400, limit_w=6000), + now=_wall_now(), + ) + # Strip the Show Debug spans — mainline must be ISO-free. + visible = re.sub(r'
]*>.*?
', "", html, flags=re.DOTALL) + assert not re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}", visible), visible + # ISO timestamps still appear in Show Debug. + assert re.search(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}", html), html + + +def test_render_fresh_applied_suppresses_simulation_only_banner(): + """Do not apply an obsolete simulation warning to verified current execution.""" + html = render_joint_site_preview( + _applied_report(now=_wall_now()), + _native_diag("ok", peak_w=5400, limit_w=6000), + now=_wall_now(), + ) + for forbidden in ("Simulation only — not executed", "EV SOC unconfirmed", "native grid limit exceeded", "Details: Show Debug"): + assert forbidden not in html, (forbidden, html) + + +def test_render_fresh_applied_suppresses_visible_native_grid_warning(): + """Keep the unrelated native counterfactual warning out of the active banner.""" + html = render_joint_site_preview( + _applied_report(now=_wall_now()), + _native_diag("exceeded", peak_w=7200, limit_w=6000, minutes_over=15), + now=_wall_now(), + ) + # Every block that mentions the native-grid diagnostic must carry + # ``display:none`` so the visible mainline never claims a current + # native-grid-limit breach once applied. + matches = re.findall( + r']*style="([^"]*)"[^>]*>(?:(?!).)*Native import limit exceeded', + html, + flags=re.DOTALL, + ) + assert matches, html + for style_attr in matches: + assert "display:none" in style_attr, style_attr + # Obsolete counterfactual lives under Show Debug only. + assert "obsolete native counterfactual" in html, html + + +def test_render_fresh_applied_no_button_or_input(): + """Displaying execution must not introduce manual actuator controls.""" + html = render_joint_site_preview( + _applied_report(now=_wall_now()), + _native_diag("ok", peak_w=5400, limit_w=6000), + now=_wall_now(), + ) + assert "= valid_until and the function should report "expired". + observed = now - timedelta(seconds=2) + valid_until = observed + timedelta(seconds=1) + report = _good_report( + now=now, + status="applied", + requested_mode="apply", + execution_enabled=True, + actuator_signals_modified=True, + execution_observed_at=observed.isoformat(), + execution_valid_until=valid_until.isoformat(), + generated_minutes_ago=0.05, + ) + reason = _applied_block_reason(report, now) + assert "expired" in reason, reason + + +def test_block_reason_inverter_observed_label_generic(): + """Do not present an observation disclaimer as a physical confirmation.""" + from joint_site_preview import _applied_block_reason + + now = _wall_now() + report = _applied_report(now=now) + report["execution_reason"] = "inverter_observed_not_hardware_confirmed" + reason = _applied_block_reason(report, now) + assert reason == "awaiting hardware confirmation", reason + + +# ---------------------------------------------------------------------- +# Gap 5: apply_blocked status lands on execution_blocked, not joint_unavailable +# ---------------------------------------------------------------------- + + +def test_apply_blocked_status_surfaces_execution_blocked(): + """An executor rejection is an execution block rather than a simulation.""" + now = _wall_now() + report = _applied_report(now=now, status_block="apply_blocked", execution_reason="apply_commissioning_required") + view = build_joint_plan_view(_native_plan(), report, now=now) + assert view.get("execution_blocked") == "apply_commissioning_required", view + assert view.get("joint_unavailable") is not True, view + assert view["joint_simulation"] is False, view + assert view["rows"] == [], view + + +def test_apply_blocked_status_with_unknown_reason_uses_coherence_reason(): + """Unknown reasons fall back to bounded evidence diagnostics.""" + now = _wall_now() + report = _applied_report(now=now, status_block="apply_blocked", execution_reason="private leak") + view = build_joint_plan_view(_native_plan(), report, now=now) + assert "private" not in view.get("execution_blocked", ""), view + assert "leak" not in view.get("execution_blocked", ""), view + + +def test_shadow_candidate_unavailable_keeps_generic_replay_path(): + """Preserve the unavailable-shadow display contract.""" + now = _wall_now() + shadow = _good_report(now=now, status="native_rejected") + view = build_joint_plan_view(_native_plan(), shadow, now=now) + assert view.get("joint_unavailable") is True, view + assert "execution_blocked" not in view, view + assert view["joint_simulation"] is True, view + + +# ---------------------------------------------------------------------- +# Receipt freshness: original gating contract, end-to-end +# ---------------------------------------------------------------------- + + +def test_fresh_applied_receipt_passes(): + """Accept a coherent and still valid current-command receipt.""" + now = _wall_now() + report = _applied_report(now=now) + assert _applied_receipt_fresh(report, now) is True + + +def test_expired_applied_receipt_rejects(): + """Reject an expired execution interval.""" + now = _wall_now() + report = _applied_report(now=now, valid_until=now - timedelta(seconds=1), observed_at=now - timedelta(seconds=61)) + assert _applied_receipt_fresh(report, now) is False + + +def test_missing_observed_at_rejects(): + """Require an explicit command observation time.""" + now = _wall_now() + report = _applied_report(now=now) + report["execution_observed_at"] = None + assert _applied_receipt_fresh(report, now) is False + + +def test_missing_valid_until_rejects(): + """Require bounded expiry rather than open-ended execution authority.""" + now = _wall_now() + report = _applied_report(now=now) + report["execution_valid_until"] = None + assert _applied_receipt_fresh(report, now) is False + + +def test_future_observed_at_rejects(): + """Future observations cannot authorise a present active-plan display.""" + now = _wall_now() + future_obs = now + timedelta(seconds=120) + report = _applied_report(now=now, observed_at=future_obs, valid_until=future_obs + timedelta(seconds=60)) + assert _applied_receipt_fresh(report, now) is False + + +def test_false_authority_rejects_execution_enabled_false(): + """Disabled execution cannot be made active by receipt fields alone.""" + now = _wall_now() + report = _applied_report(now=now, execution_enabled=False) + assert _applied_receipt_fresh(report, now) is False + + +def test_false_authority_rejects_actuator_signals_modified_false(): + """A report without executor signalling remains unexecuted.""" + now = _wall_now() + report = _applied_report(now=now, actuator_signals_modified=False) + assert _applied_receipt_fresh(report, now) is False + + +def test_wrong_requested_mode_rejects(): + """Only explicit apply mode can produce the active-plan presentation.""" + now = _wall_now() + report = _applied_report(now=now, requested_mode="shadow") + assert _applied_receipt_fresh(report, now) is False + + +def test_interval_over_300_seconds_rejects(): + """Reject execution leases exceeding the five-minute authority bound.""" + now = _wall_now() + report = _applied_report(now=now, interval_seconds=400) + assert _applied_receipt_fresh(report, now) is False + + +def test_invalid_applied_clears_rows_and_no_native_fallback(): + """Invalid execution clears rows instead of substituting the native plan.""" + now = _wall_now() + native = _native_plan() + native["rows"] = [{"time": "2026-01-01T12:00:00+0000", "import_rate": 5.0}] + report = _applied_report(now=now, valid_until=now - timedelta(seconds=1), observed_at=now - timedelta(seconds=61)) + view = build_joint_plan_view(native, report, now=now) + assert view is not native, view + assert view["rows"] == [], view + assert view.get("execution_blocked"), view + + +def test_original_table_columns_styles_totals_preserved_for_valid_applied(): + """Preserve the established table schema, colours and replay-owned totals.""" + now = _wall_now() + view = build_joint_plan_view(_native_plan(), _applied_report(now=now), now=now) + row = view["rows"][0] + for key in ( + "time", + "slot_minute", + "import_rate", + "export_rate", + "state", + "state_text", + "show_limit", + "pv_forecast", + "pv_forecast10", + "load_forecast", + "load_forecast10", + "car_charging", + "joint_ev_on_minutes", + "soc_percent", + "soc_change", + "soc_sym", + "joint_soc_end_percent", + "joint_soc_pv10_start_percent", + "joint_soc_pv10_end_percent", + "joint_grid_peak_w", + "joint_grid_peak_pv10_w", + "joint_interval_end", + "rate_color_import", + "rate_color_export", + "cost_color", + "state_color", + "soc_color", + "load_color", + "pv_color", + "car_color", + ): + assert key in row, key + assert view["totals"] == {"pv_forecast": 0.2, "load_forecast": 0.5, "car_charging": 0.0, "soc_percent": 51, "total_cost": 1.37, "clipped": 0}, view["totals"] + assert view["num_cars"] == 1, view + assert view["iboost_enable"] is False, view + assert view["carbon_enable"] is False, view + + +# ---------------------------------------------------------------------- +# Web entry points: /api/plan_data and /plan HTML +# ---------------------------------------------------------------------- + + +def test_html_api_plan_data_emits_joint_execution_for_fresh_applied(): + """The actual API route must expose the verified execution view.""" + page = _build_page(_applied_report(now=_wall_now())) + data = json.loads(asyncio.run(page.html_api_plan_data(SimpleNamespace(rel_url=SimpleNamespace(query={})))).text) + assert data["plan"]["joint_execution"] is True, data + assert data["plan"]["joint_simulation"] is False, data + assert len(data["plan"]["rows"]) == 1, data + assert "execution_blocked" not in data["plan"], data + + +def test_html_plan_emits_active_plan_banner_for_fresh_applied(): + """The actual HTML route must render a fresh active-plan report.""" + page = _build_page(_applied_report(now=_wall_now())) + html = asyncio.run(page.html_plan(SimpleNamespace(rel_url=SimpleNamespace(query={})))).text + assert "Active plan" in html, html + assert "Applied plan (executed)" not in html, html + + +def test_html_plan_emits_execution_blocked_banner_for_invalid_applied(): + """The actual HTML route must show expired execution as blocked.""" + now = _wall_now() + report = _applied_report(now=now, valid_until=now - timedelta(seconds=1), observed_at=now - timedelta(seconds=61)) + page = _build_page(report) + html = asyncio.run(page.html_plan(SimpleNamespace(rel_url=SimpleNamespace(query={})))).text + assert "Execution blocked" in html, html + assert "Active plan" not in html, html + + +# ---------------------------------------------------------------------- +# JS renderer readonly gate +# ---------------------------------------------------------------------- + + +def test_js_renderer_forces_editable_false_for_joint_execution(): + """Execution rows cannot edit unrelated native overrides through JavaScript.""" + from web_helper import get_plan_renderer_js + + js = get_plan_renderer_js() + assert "joint_execution" in js, js + assert "joint_execution !== true" in js, js + # Original shadow gate remains intact. + assert "joint_simulation !== true" in js, js + + +def test_js_renderer_does_not_advertise_simulation_only_for_applied(): + """Client-side rendering must respect the explicit execution flag.""" + from web_helper import get_plan_renderer_js + + js = get_plan_renderer_js() + assert "joint_simulation === true || data.joint_execution === true" in js, js + + +# ---------------------------------------------------------------------- +# JS renderer — whole-number SoC display in the original Plan table +# ---------------------------------------------------------------------- + + +def test_js_renderer_rounds_fractional_soc_in_plan_cell(): + """The original Plan table must display integer SoC for every row. + + The joint solver emits fractional ``battery_start_percent`` values such + as ``29.999377993406732`` for an active (executed) plan; the native + solver already rounds via ``calc_percent_limit``. The Plan table must + display whole-number percent in both cases — a previous gating of the + ``Math.round`` on ``joint_simulation`` only left the executed joint + plan (``joint_execution=true``, ``joint_simulation=false``) showing + fractional strings. Rounding is applied at render time so raw planning + SOC/math is unchanged upstream. + """ + from web_helper import get_plan_renderer_js + + js = get_plan_renderer_js() + # Row cell: every Plan row is rendered with Math.round applied + # unconditionally, including executed joint plans. + assert "${Math.round(row.soc_percent)}${socSym}" in js, js + # Totals cell: same integer rounding so totals are consistent. + assert "Math.round(totals.soc_percent" in js, js + + +# ---------------------------------------------------------------------- +# Runner +# ---------------------------------------------------------------------- + + +def run_joint_site_applied_view_tests(base): + """Registered-style runner. Parent ``unit_test.py`` wires this under the + suggested name ``joint_site_applied_view``. Each test runs in isolation + against an in-memory base — no live HA, real inverter, or actuator.""" + tests = [ + test_valid_applied_view_sets_joint_simulation_false, + test_valid_applied_view_preserves_execution_dates, + test_shadow_candidate_keeps_joint_simulation_true, + test_render_fresh_applied_uses_active_plan_label, + test_render_fresh_applied_mainline_has_no_iso_dates, + test_render_fresh_applied_suppresses_simulation_only_banner, + test_render_fresh_applied_suppresses_visible_native_grid_warning, + test_render_fresh_applied_no_button_or_input, + test_render_applied_with_missing_preview_falls_back_to_execution_blocked, + test_render_applied_with_stale_generated_at_falls_back_to_execution_blocked, + test_render_applied_with_stale_soc_hypothesis_falls_back_to_execution_blocked, + test_render_applied_with_counterfactual_controls_falls_back_to_execution_blocked, + test_render_applied_with_failed_paired_checks_falls_back_to_execution_blocked, + test_execution_blockers_non_list_rejects_fresh, + test_execution_blockers_nonempty_list_rejects_fresh, + test_execution_blockers_empty_list_accepts_fresh, + test_execution_blockers_absent_accepts_fresh, + test_block_reason_echoes_only_allowlisted_labels, + test_block_reason_drops_unknown_reason_strings, + test_block_reason_uses_now_check_when_no_reason, + test_block_reason_inverter_observed_label_generic, + test_apply_blocked_status_surfaces_execution_blocked, + test_apply_blocked_status_with_unknown_reason_uses_coherence_reason, + test_shadow_candidate_unavailable_keeps_generic_replay_path, + test_fresh_applied_receipt_passes, + test_expired_applied_receipt_rejects, + test_missing_observed_at_rejects, + test_missing_valid_until_rejects, + test_future_observed_at_rejects, + test_false_authority_rejects_execution_enabled_false, + test_false_authority_rejects_actuator_signals_modified_false, + test_wrong_requested_mode_rejects, + test_interval_over_300_seconds_rejects, + test_invalid_applied_clears_rows_and_no_native_fallback, + test_original_table_columns_styles_totals_preserved_for_valid_applied, + test_html_api_plan_data_emits_joint_execution_for_fresh_applied, + test_html_plan_emits_active_plan_banner_for_fresh_applied, + test_html_plan_emits_execution_blocked_banner_for_invalid_applied, + test_js_renderer_forces_editable_false_for_joint_execution, + test_js_renderer_does_not_advertise_simulation_only_for_applied, + test_js_renderer_rounds_fractional_soc_in_plan_cell, + ] + failed = False + for test in tests: + print("Test: " + test.__name__) + try: + test() + except Exception as exc: + print("ERROR: {}: {}".format(test.__name__, exc)) + failed = True + else: + print("PASS: " + test.__name__) + return failed diff --git a/apps/predbat/tests/test_joint_site_apply.py b/apps/predbat/tests/test_joint_site_apply.py new file mode 100644 index 000000000..045a5859a --- /dev/null +++ b/apps/predbat/tests/test_joint_site_apply.py @@ -0,0 +1,1169 @@ +"""Offline acceptance of the gated joint-site native execution path. + +These tests verify the apply-side behaviour of :mod:`joint_site_apply` against +the rewritten contract: + +* ``execute_joint_site(base)`` returns ``None`` for off/shadow, otherwise a + ``(status, status_extra)`` tuple or the literal ``("Joint blocked", reason)`` + pair returned from the blocked branch. +* ``publish_joint_ev(base)`` returns ``False`` for off/shadow and ``True`` for + apply (the fetch-time path always sees a True return so the parent executor + can skip its legacy EV publication). +* ``execution_fingerprint(base)`` hashes the actual live configuration; it + never reads from ``base.joint_site_pending``. +* ``ready_to_activate`` and ``hardware_confirmed`` are always ``False`` — + HA observations never constitute hardware confirmation. +* The only actuator permitted is ``inverter.adjust_charge_immediate(...)``. + ``adapter.apply`` is a deprecated bypass that must never be called. + +Tests build a real ``base`` via the parent's :func:`_apply_inputs` fixture +(which configures the EV session, entity mapping and commissioning flag), stage +a candidate through the real ``evaluate_joint_site`` solver/replay path, and +inject a fake ``DEYE`` inverter with a controllable ``adjust_charge_immediate`` +spy plus a fake adapter that exposes only ``snapshot`` and ``settings``. + +The unit suite never calls a real HA service, never writes through the live +DEYE adapter, and never depends on a connected EVSE. All evidence comes from +the ``base.joint_site_report`` / ``base.joint_site_execution`` / dashboard +sensor surface that the executor publishes. +""" + +import json +from copy import deepcopy +from datetime import timedelta, timezone +from types import SimpleNamespace +from unittest.mock import patch + +from const import PREDBAT_MAX_CARS + +import joint_site_apply +import joint_site_native +from joint_site_apply import ( + OBSERVATION_MAX_AGE_SECONDS, + RECEIPT_TTL_SECONDS, + execute_joint_site, + execution_fingerprint, + publish_joint_ev, +) +from joint_site_native import evaluate_joint_site +from tests.test_joint_site_entrypoints import _apply_inputs +from tests.test_joint_site_native import _setup + + +# ---------------------------------------------------------------------- +# Fixture helpers — own clock, own field reset, real _setup + _apply_inputs. +# ---------------------------------------------------------------------- + + +def _now_utc(base): + """Return the fixture's pinned ``now_utc`` as an offset-aware datetime.""" + if base.now_utc.tzinfo is None: + return base.now_utc.replace(tzinfo=timezone.utc) + return base.now_utc + + +def _set_entity(base, entity_id, state, last_updated=None, last_changed=None): + """Populate the TestHAInterface cache with a standard HA shape.""" + moment = _now_utc(base) + base.ha_interface.dummy_items[entity_id] = { + "state": state, + "last_updated": last_updated if last_updated is not None else moment, + "last_changed": last_changed if last_changed is not None else moment, + } + + +def _stale_or_future_entity(base, entity_id, last_updated, last_changed=None): + """Helper that intentionally sets an entity with an out-of-window stamp. + + The apply path parses the observation stamp from ``state``, so we rewrite + the ISO timestamp there too — just adjusting ``last_updated`` would leave + the freshness check passing. + """ + base.ha_interface.dummy_items[entity_id] = { + "state": last_updated.isoformat() if hasattr(last_updated, "isoformat") else last_updated, + "last_updated": last_updated, + "last_changed": last_changed if last_changed is not None else last_updated, + } + + +def _make_adapter(control=True, hold_verified=True, cap_w=8192): + """Build a controllable DEYE adapter stand-in. + + The adapter exposes only ``snapshot`` and ``settings`` per the rewritten + apply contract; the actuator path lives on the inverter, not here. + """ + + snapshot_calls = {"count": 0} + + def _snapshot(): + """Return a deterministic fake snapshot; tracks invocations for assertions.""" + snapshot_calls["count"] += 1 + return {"ok": True} + + adapter = SimpleNamespace( + settings={ + "control": control, + "hold_verified": hold_verified, + "max_discharge_power_w": cap_w, + }, + snapshot=_snapshot, + ) + return adapter, snapshot_calls + + +def _make_inverter(base, adapter, *, inverter_type="DEYE", supports_freeze=True, in_calibration=False, observed_return=True, log=None): + """Build a fake inverter with the surface the new apply contract reads.""" + spy_calls = {"args": []} + + def _adjust(target_soc, freeze=False, discharge_power_w=None): + """Record the dispatch call and return ``observed_return`` for the executor.""" + spy_calls["args"].append( + { + "target_soc": target_soc, + "freeze": freeze, + "discharge_power_w": discharge_power_w, + } + ) + return observed_return + + inverter = SimpleNamespace( + id=0, + inverter_type=inverter_type, + inv_support_charge_freeze=supports_freeze, + in_calibration=in_calibration, + deye_local=adapter, + battery_rate_max_discharge=base.battery_rate_max_discharge, + adjust_charge_immediate=_adjust, + log=log if log is not None else (lambda *_args, **_kwargs: None), + ) + base.inverters = [inverter] + return inverter, spy_calls + + +def _reset_apply_fields(base): + """Clear transient apply-related attributes so each test starts clean.""" + base.joint_site_pending = None + base.joint_site_execution = None + base.joint_site_report = None + base.inverter_rate_intent = {} + base.isCharging = False + base.isExporting = False + base.car_charging_slots = [[] for _ in range(PREDBAT_MAX_CARS)] + base.set_read_only = False + base.set_read_only_axle = False + base.inverter_needs_reset = False + base.had_errors = False + base.fatal_error = False + base.manual_soc_max_keep = None + base.manual_charge_times = [] + base.manual_demand_times = [] + base.manual_export_times = [] + base.manual_freeze_charge_times = [] + base.manual_freeze_export_times = [] + base.all_active_keep = {} + base.all_active_keep_max = {} + base.io_adjusted = {} + if hasattr(base, "ha_interface") and base.ha_interface is not None: + for entity_id in ( + "binary_sensor.test_relay", + "sensor.test_ev_soc", + "sensor.test_charger_state", + "switch.test_permission", + "sensor.test_connection_observed_at", + "sensor.test_soc_observed_at", + ): + base.ha_interface.dummy_items.pop(entity_id, None) + + +def _stage_apply(base, *, pv=5, minutes=60, mode="apply", apply_verified=True): + """Configure the apply fixture via the parent's _apply_inputs helper.""" + _setup(base, pv=pv, minutes=minutes) + _apply_inputs(base) + if mode != "apply": + base.args["joint_site"]["mode"] = mode + if not apply_verified: + base.args["joint_site"]["apply_verified"] = False + elif "apply_verified" in base.args["joint_site"]: + base.args["joint_site"]["apply_verified"] = True + + +def _evaluate(base, *, pv=5, minutes=60): + """Run the real solver and replay path so the candidate is genuine.""" + _stage_apply(base, pv=pv, minutes=minutes) + with patch.object(joint_site_native, "_observation_clock", side_effect=lambda: _now_utc(base)): + return evaluate_joint_site(base, publish=False) + + +def _freeze_command_ev(pending, ev_on): + """Mutate the staged first command's ``ev_on`` flag for the EV-on cases.""" + if not isinstance(pending, dict): + return False + commands = pending.get("commands") or [] + if not commands: + return False + commands[0]["ev_on"] = bool(ev_on) + return True + + +# ---------------------------------------------------------------------- +# Test cases — ordered from the broadest noop through to verification of +# receipt/observability. Each case runs inside the run-loop's clock+reset +# patches so observable state can be asserted cleanly. +# ---------------------------------------------------------------------- + + +def test_off_and_shadow_are_strict_noop(base): + """Off and shadow must return ``None`` from execute_joint_site and never reach the inverter.""" + _stage_apply(base) + for mode in ("off", "shadow"): + _evaluate(base) + base.args["joint_site"]["mode"] = mode + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + before_intent = dict(base.inverter_rate_intent) + before_execution = deepcopy(getattr(base, "joint_site_execution", None)) + before_report = deepcopy(getattr(base, "joint_site_report", None)) + before_slots = deepcopy(base.car_charging_slots) + assert execute_joint_site(base) is None, mode + assert spy["args"] == [], mode + assert base.inverter_rate_intent == before_intent, mode + assert getattr(base, "joint_site_execution", None) == before_execution, mode + assert getattr(base, "joint_site_report", None) == before_report, mode + assert base.car_charging_slots == before_slots, mode + assert publish_joint_ev(base) is False, mode + + +def test_apply_alone_rejected_without_verified(base): + """Apply without the commissioning flag is a hard no-actuate gate.""" + report = _evaluate(base) + assert report["status"] == "candidate", report + base.args["joint_site"]["apply_verified"] = False + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["status"] == "apply_blocked", base.joint_site_report + assert base.joint_site_report["execution_enabled"] is False, base.joint_site_report + assert base.joint_site_report["execution_reason"] == "apply_commissioning_required" + assert base.joint_site_report["ready_to_activate"] is False + assert base.joint_site_report["hardware_confirmed"] is False + + +def test_apply_blocked_when_inverter_type_sk(base): + """A non-DEYE inverter type is a hard gate; ``adjust_charge_immediate`` never runs.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter, inverter_type="SK") + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "local_deye_required" + assert base.joint_site_report["execution_enabled"] is False + assert base.joint_site_report["hardware_confirmed"] is False + + +def test_apply_blocked_when_deye_adapter_missing(base): + """A DEYE inverter without ``deye_local`` cannot actuate.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + inverter.deye_local = None + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "local_deye_adapter_missing" + + +def test_apply_blocked_when_hold_not_verified(base): + """``hold_verified`` must be exactly True on the adapter settings.""" + _evaluate(base) + adapter, _ = _make_adapter(hold_verified=False) + inverter, spy = _make_inverter(base, adapter) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "deye_commissioning_required" + + +def test_apply_blocked_when_control_flag_false(base): + """``control`` must be exactly True on the adapter settings.""" + _evaluate(base) + adapter, _ = _make_adapter(control=False) + inverter, spy = _make_inverter(base, adapter) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "deye_commissioning_required" + + +def test_apply_blocked_when_inverter_lacks_freeze_support(base): + """An inverter that does not declare freeze support must not actuate.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter, supports_freeze=False) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "inverter_capability_or_calibration" + + +def test_apply_blocked_when_inverter_in_calibration(base): + """``in_calibration=True`` blocks apply regardless of every other condition.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter, in_calibration=True) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "inverter_capability_or_calibration" + + +def test_apply_blocked_when_read_only(base): + """``set_read_only`` (and the axle variant) gate apply without invoking the inverter.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + for attr in ("set_read_only", "set_read_only_axle"): + base.__setattr__(attr, True) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", (attr, result) + assert spy["args"] == [], attr + assert base.joint_site_report["execution_reason"] == "read_only", attr + base.__setattr__(attr, False) + + +def test_apply_blocked_when_native_blockers_present(base): + """A manual override, active keep or io override produces a native blocker.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.manual_charge_times = [{"start": base.minutes_now, "end": base.minutes_now + 30}] + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "native_inputs_or_overrides_blocked" + base.manual_charge_times = [] + base.all_active_keep_max = {base.minutes_now: 50.0} + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert base.joint_site_report["execution_reason"] == "native_inputs_or_overrides_blocked" + + +def test_apply_blocked_by_booting_calibration_flags(base): + """``inverter_needs_reset`` / ``had_errors`` / ``fatal_error`` are native state.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.inverter_needs_reset = True + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "native_state_not_ready" + base.inverter_needs_reset = False + base.had_errors = True + result = execute_joint_site(base) + assert base.joint_site_report["execution_reason"] == "native_state_not_ready" + + +def test_apply_blocked_when_deye_dispatch_model_mismatch(base): + """``export_limit != 0`` or ``set_discharge_during_charge=False`` is a hard mismatch.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.export_limit = 0.05 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "deye_dispatch_model_mismatch" + # Reset had_errors so the next call isn't short-circuited by native state. + base.had_errors = False + base.export_limit = 0 + base.set_discharge_during_charge = False + result = execute_joint_site(base) + assert base.joint_site_report["execution_reason"] == "deye_dispatch_model_mismatch" + + +def test_apply_blocked_when_manual_ceiling_unsupported(base): + """``manual_soc_max_keep`` is incompatible with native apply.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.manual_soc_max_keep = 80.0 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "manual_ceiling_unsupported" + + +def test_apply_blocked_by_missing_relay_entity(base): + """Missing relay entity observation fails ``entity_observation_unavailable``.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.ha_interface.dummy_items.pop("binary_sensor.test_relay", None) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "entity_observation_unavailable", + "native_inputs_or_overrides_blocked", + }, base.joint_site_report + + +def test_apply_blocked_by_missing_permission_entity(base): + """Missing permission entity observation fails ``entity_observation_unavailable``.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.ha_interface.dummy_items.pop("switch.test_permission", None) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "entity_observation_unavailable", + "native_inputs_or_overrides_blocked", + }, base.joint_site_report + + +def test_apply_blocked_by_permission_off(base): + """``switch.test_permission == 'off'`` is rejected.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + _set_entity(base, "switch.test_permission", "off") + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "permission_not_granted" + + +def test_apply_blocked_by_missing_connection_entity(base): + """Missing connection entity observation fails ``entity_observation_unavailable``.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.ha_interface.dummy_items.pop("sensor.test_charger_state", None) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "entity_observation_unavailable", + "native_inputs_or_overrides_blocked", + }, base.joint_site_report + + +def test_apply_blocked_by_connection_state_allowlist(base): + """A connection state not in the configured allowlist fails ``connection_not_confirmed``.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + _set_entity(base, "sensor.test_charger_state", "3") + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "connection_not_confirmed" + + +def test_apply_blocked_by_missing_timestamp_entity(base): + """Missing soc/connection observation timestamp entity fails the freshness gate.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.ha_interface.dummy_items.pop("sensor.test_soc_observed_at", None) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "entity_observation_unavailable", + "native_inputs_or_overrides_blocked", + }, base.joint_site_report + + +def test_apply_blocked_by_stale_soc_observation(base): + """An soc observation older than ``OBSERVATION_MAX_AGE_SECONDS`` is rejected.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + stale = _now_utc(base) - timedelta(seconds=OBSERVATION_MAX_AGE_SECONDS + 60) + _stale_or_future_entity(base, "sensor.test_soc_observed_at", stale) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "physical_observation_stale_or_future" + + +def test_apply_blocked_by_future_connection_observation(base): + """A future connection observation is invalid even with a fresh last_updated.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + future = _now_utc(base) + timedelta(seconds=120) + _stale_or_future_entity(base, "sensor.test_connection_observed_at", future) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "physical_observation_stale_or_future" + + +def test_apply_blocked_by_future_relay_timestamp(base): + """``last_updated > now`` on the relay entity is a future timestamp.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + future = _now_utc(base) + timedelta(seconds=300) + # Override only ``last_updated`` / ``last_changed``; the relay state itself + # must remain a valid 'on'/'off' string for the solver to accept the entity. + relay_payload = base.ha_interface.dummy_items["binary_sensor.test_relay"] + relay_payload["last_updated"] = future + relay_payload["last_changed"] = future + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert base.joint_site_report["execution_reason"] in { + "entity_timestamp_future", + "native_inputs_or_overrides_blocked", + }, base.joint_site_report + + +def test_apply_blocked_by_cooldown_within_599_seconds(base): + """599s after the relay flipped OFF the cooldown blocks an ev_on request.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + recent_off = _now_utc(base) - timedelta(seconds=599) + # Only override the relay's last_changed; the state string must remain "off". + payload = base.ha_interface.dummy_items["binary_sensor.test_relay"] + payload["last_changed"] = recent_off + payload["last_updated"] = recent_off + timedelta(seconds=1) + _freeze_command_ev(base.joint_site_pending, ev_on=True) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "ev_cooldown" + + +def test_apply_allowed_after_cooldown_600_seconds(base): + """Exactly 600s after the relay flipped OFF the cooldown is satisfied.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + long_off = _now_utc(base) - timedelta(seconds=600) + payload = base.ha_interface.dummy_items["binary_sensor.test_relay"] + payload["last_changed"] = long_off + payload["last_updated"] = long_off + timedelta(seconds=1) + _freeze_command_ev(base.joint_site_pending, ev_on=True) + result = execute_joint_site(base) + assert spy["args"], result + assert result[0] in ("Demand", "Charging", "Freeze charging"), result + + +def test_apply_freeze_skips_cooldown_entirely(base): + """Freeze commands bypass the cooldown gate; ev_on=False is irrelevant here.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + recent_off = _now_utc(base) - timedelta(seconds=30) + payload = base.ha_interface.dummy_items["binary_sensor.test_relay"] + payload["last_changed"] = recent_off + payload["last_updated"] = recent_off + timedelta(seconds=1) + # Freeze + ev_on=False bypasses the cooldown entirely (no EV energising). + base.joint_site_pending["commands"][0]["freeze"] = True + base.joint_site_pending["commands"][0]["target_soc"] = 0 + base.joint_site_pending["commands"][0]["ev_on"] = False + result = execute_joint_site(base) + assert spy["args"], result + assert result[0] == "Freeze charging", result + + +def test_apply_blocked_by_relay_state_invalid(base): + """``relay state != on/off`` fails ``relay_state_invalid``.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + _set_entity(base, "binary_sensor.test_relay", "unknown") + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "relay_state_invalid", + "execution_validation_failed", + }, base.joint_site_report + + +def test_apply_blocked_by_import_limit_mismatch(base): + """A change in ``base.import_limit`` after staging invalidates the fingerprint.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.import_limit += 200.0 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "configuration_changed", + "configuration_changed_during_dispatch", + "import_limit_mismatch", + }, base.joint_site_report + + +def test_apply_blocked_by_manual_floor_violation(base): + """``soc_kw`` below the active keep floor is a domestic floor violation.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.all_active_keep = {base.minutes_now: 80.0} + base.soc_kw = 5.0 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "domestic_manual_floor", + "native_inputs_or_overrides_blocked", + }, base.joint_site_report + + +def test_apply_blocked_by_invalid_power_below_zero(base): + """``power_w < 0`` is rejected; the cap does not clamp the rejection.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"][0]["power_w"] = -100 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "discharge_power_out_of_range" + + +def test_apply_blocked_by_power_above_8192(base): + """``power_w > 8192`` is rejected — no clamping to the cap.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"][0]["power_w"] = 9000 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "discharge_power_out_of_range" + + +def test_apply_blocked_by_nonfinite_power(base): + """``power_w = float('inf')`` is rejected.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"][0]["power_w"] = float("inf") + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "discharge_power_out_of_range" + + +def test_apply_blocked_by_bool_power(base): + """``power_w is True`` is rejected (booleans are not accepted as numbers).""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"][0]["power_w"] = True + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "discharge_power_out_of_range" + + +def test_apply_blocked_by_invalid_target_soc(base): + """``target_soc`` not in {0, 100} is rejected.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"][0]["target_soc"] = 50 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "invalid_target" + + +def test_apply_blocked_by_freeze_with_nonzero_target(base): + """``freeze=True`` together with ``target_soc != 0`` is rejected.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"][0]["freeze"] = True + base.joint_site_pending["commands"][0]["target_soc"] = 100 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "invalid_hold_target" + + +def test_apply_blocked_by_nonboolean_freeze_flag(base): + """``freeze`` not exactly bool is rejected.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"][0]["freeze"] = 1 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "invalid_command_boolean" + + +def test_apply_blocked_by_command_timeline_mismatch(base): + """A command whose ``start`` is not the first planned step is rejected.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"][0]["start"] = base.minutes_now + 5 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "command_timeline_mismatch" + + +def test_apply_blocked_by_wallclock_crossing_command_end(base): + """``now >= last command end`` (``command_authority_expired``) blocks apply. + + Without changing ``minutes_now`` (the parent solver bound) we mutate the + staged command's end to force the expiry check, simulating a wallclock + that crossed the command boundary during a dispatch attempt. + """ + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"][0]["end"] = base.minutes_now + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "command_expired_during_dispatch", + "current_command_expired", + "command_timeline_mismatch", + }, base.joint_site_report + + +def test_apply_blocked_by_no_pending_candidate(base): + """``base.joint_site_pending = None`` triggers ``no_pending_candidate``.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending = None + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "no_pending_candidate" + + +def test_apply_blocked_by_native_check_failure_in_report(base): + """A pending whose report status is not ``candidate`` is rejected.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["report"]["status"] = "native_rejected" + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "candidate_not_accepted" + + +def test_apply_blocked_by_stale_candidate_generated_at(base): + """A pending whose ``generated_at`` is older than TTL is stale.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + stale = _now_utc(base) - timedelta(seconds=RECEIPT_TTL_SECONDS + 60) + base.joint_site_pending["report"]["generated_at"] = stale.isoformat() + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "candidate_stale_or_future" + + +def test_apply_blocked_by_changed_configuration_during_dispatch(base): + """A fingerprint mismatch between the staged candidate and the present base blocks apply.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["fingerprint"] = "deadbeef" * 8 + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "configuration_changed", + "configuration_changed_during_dispatch", + }, base.joint_site_report + + +def test_apply_blocked_by_incomplete_command_horizon(base): + """A horizon that does not cover every PREDICT_STEP interval is rejected.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + base.joint_site_pending["commands"] = base.joint_site_pending["commands"][:2] + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] in { + "incomplete_command_horizon", + "native_checks_failed", + }, base.joint_site_report + + +def test_apply_blocked_when_inverter_returns_false(base): + """``adjust_charge_immediate`` returning ``False`` is a no-actuate result.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter, observed_return=False) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] + assert base.joint_site_report["execution_reason"] == "inverter_command_not_observed" + assert base.joint_site_report["execution_enabled"] is False + + +def test_success_demand_charge_hold_statuses(base): + """Demand, charge and freeze each produce the matching status and call the inverter.""" + cases = [ + {"target_soc": 0, "freeze": False, "expected": "Demand"}, + {"target_soc": 100, "freeze": False, "expected": "Charging"}, + {"target_soc": 0, "freeze": True, "expected": "Freeze charging"}, + {"target_soc": 0, "freeze": False, "expected": "Demand", "power_w": 4000}, + ] + for case in cases: + _evaluate(base) + base.joint_site_pending["commands"][0]["target_soc"] = case["target_soc"] + base.joint_site_pending["commands"][0]["freeze"] = case["freeze"] + base.joint_site_pending["commands"][0]["ev_on"] = False + if "power_w" in case: + base.joint_site_pending["commands"][0]["power_w"] = case["power_w"] + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + result = execute_joint_site(base) + assert spy["args"], (case, result) + assert result[0] == case["expected"], (case, result) + report = base.joint_site_report + assert report["execution_enabled"] is True, (case, report) + assert report["ready_to_activate"] is False, (case, report) + assert report["hardware_confirmed"] is False, (case, report) + assert report["status"] == "applied", (case, report) + if "power_w" in case: + assert spy["args"][-1]["discharge_power_w"] == case["power_w"], (case, spy["args"]) + + +def test_stale_observation_state_blocks_even_with_fresh_last_updated(base): + """A stale ``state`` value on the observation entity blocks the freshness gate + even when HA reports ``last_updated`` as freshly observed. + """ + _evaluate(base) + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + # Construct an entity whose last_updated is fresh but whose state string is + # a stale ISO timestamp — the apply contract reads the stamp from state. + stale_state = (_now_utc(base) - timedelta(seconds=OBSERVATION_MAX_AGE_SECONDS + 60)).isoformat() + payload = base.ha_interface.dummy_items["sensor.test_soc_observed_at"] + payload["state"] = stale_state + payload["last_updated"] = _now_utc(base) + payload["last_changed"] = _now_utc(base) + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert spy["args"] == [] + assert base.joint_site_report["execution_reason"] == "physical_observation_stale_or_future" + + +def test_pending_false_and_exception_revokes_old_receipt(base): + """A pending None and an actuator exception together revoke the prior receipt.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + if base.minutes_now not in base.rate_import: + base.rate_import[base.minutes_now] = 10.0 + # Accept a real receipt first. + execute_joint_site(base) + assert isinstance(base.joint_site_execution, dict) and base.joint_site_execution.get("observations_match") + # Now drop the pending and force the actuator to raise. + base.joint_site_pending = None + # Replace inverter with one that raises from adjust_charge_immediate. Since + # pending is None, the path will block before the actuator, but the receipt + # must be cleared and the EV slot must publish OFF. + result = execute_joint_site(base) + assert isinstance(result, tuple) and result[0] == "Joint blocked", result + assert base.joint_site_execution is None + assert base.car_charging_slots == [[] for _ in range(PREDBAT_MAX_CARS)] + sensor_key = "binary_sensor." + base.prefix + "_car_charging_slot" + item = base.dashboard_values.get(sensor_key) + assert item is not None + assert item["state"] == "off" + assert item["attributes"]["ready_to_activate"] is False + assert item["attributes"]["hardware_confirmed"] is False + + +def test_success_with_ev_on_publishes_on(base): + """An accepted command and ``ev_on=True`` flips the EV slot binary sensor ON.""" + _evaluate(base) + base.joint_site_pending["commands"][0]["ev_on"] = True + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + # The runtime below uses base.minutes_now as the lookup key into rate_import. + if base.minutes_now not in base.rate_import: + base.rate_import[base.minutes_now] = 10.0 + result = execute_joint_site(base) + assert isinstance(result, tuple), result + sensor_key = "binary_sensor." + base.prefix + "_car_charging_slot" + item = base.dashboard_values.get(sensor_key) + assert item is not None + assert item["state"] == "on", item + assert item["attributes"]["ready_to_activate"] is False + assert item["attributes"]["hardware_confirmed"] is False + + +def test_success_with_ev_off_publishes_off(base): + """An accepted command and ``ev_on=False`` flips the EV slot binary sensor OFF.""" + _evaluate(base) + base.joint_site_pending["commands"][0]["ev_on"] = False + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + result = execute_joint_site(base) + assert isinstance(result, tuple), result + sensor_key = "binary_sensor." + base.prefix + "_car_charging_slot" + item = base.dashboard_values.get(sensor_key) + assert item is not None + assert item["state"] == "off", item + assert item["attributes"]["ready_to_activate"] is False + assert item["attributes"]["hardware_confirmed"] is False + + +def test_publish_joint_ev_after_accepted_publishes_immediately(base): + """The native EV publication must happen in the same cycle as the accept.""" + _evaluate(base) + base.joint_site_pending["commands"][0]["ev_on"] = True + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + if base.minutes_now not in base.rate_import: + base.rate_import[base.minutes_now] = 10.0 + execute_joint_site(base) + handled = publish_joint_ev(base) + assert handled is True + sensor_key = "binary_sensor." + base.prefix + "_car_charging_slot" + item = base.dashboard_values.get(sensor_key) + assert item is not None + assert item["state"] == "on", item + + +def test_publish_joint_ev_after_ttl_revokes_old_receipt(base): + """A receipt older than TTL is revoked on the next publish attempt.""" + _evaluate(base) + base.joint_site_pending["commands"][0]["ev_on"] = True + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + if base.minutes_now not in base.rate_import: + base.rate_import[base.minutes_now] = 10.0 + execute_joint_site(base) + # Move the receipt back past TTL. + base.joint_site_execution["generated_at"] = (_now_utc(base) - timedelta(seconds=RECEIPT_TTL_SECONDS + 30)).isoformat() + handled = publish_joint_ev(base) + assert handled is True + assert base.joint_site_execution is None + sensor_key = "binary_sensor." + base.prefix + "_car_charging_slot" + item = base.dashboard_values.get(sensor_key) + assert item is not None + assert item["state"] == "off", item + + +def test_publish_joint_ev_with_permission_off_revokes(base): + """Permission flipping OFF between accept and publish revokes the receipt.""" + _evaluate(base) + base.joint_site_pending["commands"][0]["ev_on"] = True + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + if base.minutes_now not in base.rate_import: + base.rate_import[base.minutes_now] = 10.0 + execute_joint_site(base) + _set_entity(base, "switch.test_permission", "off") + handled = publish_joint_ev(base) + assert handled is True + assert base.joint_site_execution is None + + +def test_publish_joint_ev_with_disconnect_revokes(base): + """Connection dropping out of the allowlist revokes the receipt.""" + _evaluate(base) + base.joint_site_pending["commands"][0]["ev_on"] = True + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + if base.minutes_now not in base.rate_import: + base.rate_import[base.minutes_now] = 10.0 + execute_joint_site(base) + _set_entity(base, "sensor.test_charger_state", "unplugged") + handled = publish_joint_ev(base) + assert handled is True + assert base.joint_site_execution is None + + +def test_publish_joint_ev_with_future_window_revokes(base): + """A receipt whose command window is entirely in the future is revoked.""" + _evaluate(base) + base.joint_site_pending["commands"][0]["ev_on"] = True + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + if base.minutes_now not in base.rate_import: + base.rate_import[base.minutes_now] = 10.0 + execute_joint_site(base) + base.joint_site_execution["command"]["start"] = base.minutes_now + 60 + base.joint_site_execution["command"]["end"] = base.minutes_now + 120 + handled = publish_joint_ev(base) + assert handled is True + assert base.joint_site_execution is None + + +def test_publish_joint_ev_off_shadow_returns_false(base): + """The publish path returns ``False`` for off/shadow without acting.""" + for mode in ("off", "shadow"): + _evaluate(base) + base.args["joint_site"]["mode"] = mode + handled = publish_joint_ev(base) + assert handled is False, mode + + +def test_execute_returns_tuple_when_apply(base): + """``execute_joint_site`` returns a tuple on accept and on every blocked path.""" + _evaluate(base) + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + if base.minutes_now not in base.rate_import: + base.rate_import[base.minutes_now] = 10.0 + ok_result = execute_joint_site(base) + assert isinstance(ok_result, tuple) and len(ok_result) == 2, ok_result + + _evaluate(base) + base.args["joint_site"]["apply_verified"] = False + blocked_result = execute_joint_site(base) + assert isinstance(blocked_result, tuple) and len(blocked_result) == 2, blocked_result + assert blocked_result[0] == "Joint blocked", blocked_result + + +def test_execution_fingerprint_is_configuration_independent_of_pending(base): + """The fingerprint hashes the live configuration, not the pending object.""" + _evaluate(base) + base.joint_site_pending = None + first = execution_fingerprint(base) + _evaluate(base) + base.joint_site_pending = None + second = execution_fingerprint(base) + assert first == second, "Fingerprint must be stable for identical configuration" + assert first, "Fingerprint must be non-empty for a populated base" + + +def test_inverter_rate_intent_cleared_on_apply(base): + """Any legacy quick-rebalance intent is cleared so apply's commands are not clobbered.""" + _evaluate(base) + base.joint_site_pending["commands"][0]["ev_on"] = False + adapter, _ = _make_adapter() + inverter, _ = _make_inverter(base, adapter) + base.inverter_rate_intent = {0: {"charge_rate": 1500, "discharge_rate": 0}} + if base.minutes_now not in base.rate_import: + base.rate_import[base.minutes_now] = 10.0 + execute_joint_site(base) + assert base.inverter_rate_intent == {} + + +def test_exception_in_joint_site_publish_does_not_leak_private_strings(base): + """A blocked path must not echo the underlying exception's private contents.""" + _evaluate(base) + base.joint_site_pending["commands"][0]["target_soc"] = 50 # invalid + adapter, _ = _make_adapter() + inverter, spy = _make_inverter(base, adapter) + result = execute_joint_site(base) + assert isinstance(result, tuple), result + report = base.joint_site_report + reason = report.get("execution_reason") or "" + assert "private" not in reason, reason + assert "secret" not in reason, reason + assert "password" not in reason, reason + assert "50" not in reason, reason + assert spy["args"] == [] + + +def test_evaluate_joint_site_real_solver_keeps_candidate_status(base): + """``evaluate_joint_site`` in apply mode produces a real candidate with non-empty fingerprint.""" + report = _evaluate(base, pv=5, minutes=60) + assert report["status"] == "candidate", report + assert report.get("proposed_ev_windows") is not None + assert base.joint_site_pending is not None + assert base.joint_site_pending["fingerprint"], "Fingerprint must be non-empty after staging" + assert json.dumps(base.joint_site_pending["fingerprint"]).strip('"') != "" + + +# ---------------------------------------------------------------------- +# Runner — apply the parent's clock, reset transient fields, then iterate. +# ---------------------------------------------------------------------- + + +def run_joint_site_apply_tests(base): + """Run apply execution cases without a live HA connection or inverter writes.""" + tests = [ + test_off_and_shadow_are_strict_noop, + test_apply_alone_rejected_without_verified, + test_apply_blocked_when_inverter_type_sk, + test_apply_blocked_when_deye_adapter_missing, + test_apply_blocked_when_hold_not_verified, + test_apply_blocked_when_control_flag_false, + test_apply_blocked_when_inverter_lacks_freeze_support, + test_apply_blocked_when_inverter_in_calibration, + test_apply_blocked_when_read_only, + test_apply_blocked_when_native_blockers_present, + test_apply_blocked_by_booting_calibration_flags, + test_apply_blocked_when_deye_dispatch_model_mismatch, + test_apply_blocked_when_manual_ceiling_unsupported, + test_apply_blocked_by_missing_relay_entity, + test_apply_blocked_by_missing_permission_entity, + test_apply_blocked_by_permission_off, + test_apply_blocked_by_missing_connection_entity, + test_apply_blocked_by_connection_state_allowlist, + test_apply_blocked_by_missing_timestamp_entity, + test_apply_blocked_by_stale_soc_observation, + test_apply_blocked_by_future_connection_observation, + test_apply_blocked_by_future_relay_timestamp, + test_apply_blocked_by_cooldown_within_599_seconds, + test_apply_allowed_after_cooldown_600_seconds, + test_apply_freeze_skips_cooldown_entirely, + test_apply_blocked_by_relay_state_invalid, + test_apply_blocked_by_import_limit_mismatch, + test_apply_blocked_by_manual_floor_violation, + test_apply_blocked_by_invalid_power_below_zero, + test_apply_blocked_by_power_above_8192, + test_apply_blocked_by_nonfinite_power, + test_apply_blocked_by_bool_power, + test_apply_blocked_by_invalid_target_soc, + test_apply_blocked_by_freeze_with_nonzero_target, + test_apply_blocked_by_nonboolean_freeze_flag, + test_apply_blocked_by_command_timeline_mismatch, + test_apply_blocked_by_wallclock_crossing_command_end, + test_apply_blocked_by_no_pending_candidate, + test_apply_blocked_by_native_check_failure_in_report, + test_apply_blocked_by_stale_candidate_generated_at, + test_apply_blocked_by_changed_configuration_during_dispatch, + test_apply_blocked_by_incomplete_command_horizon, + test_apply_blocked_when_inverter_returns_false, + test_success_demand_charge_hold_statuses, + test_stale_observation_state_blocks_even_with_fresh_last_updated, + test_pending_false_and_exception_revokes_old_receipt, + test_success_with_ev_on_publishes_on, + test_success_with_ev_off_publishes_off, + test_publish_joint_ev_after_accepted_publishes_immediately, + test_publish_joint_ev_after_ttl_revokes_old_receipt, + test_publish_joint_ev_with_permission_off_revokes, + test_publish_joint_ev_with_disconnect_revokes, + test_publish_joint_ev_with_future_window_revokes, + test_publish_joint_ev_off_shadow_returns_false, + test_execute_returns_tuple_when_apply, + test_execution_fingerprint_is_configuration_independent_of_pending, + test_inverter_rate_intent_cleared_on_apply, + test_exception_in_joint_site_publish_does_not_leak_private_strings, + test_evaluate_joint_site_real_solver_keeps_candidate_status, + ] + for test in tests: + _reset_apply_fields(base) + # Pin both the solver's observation clock and the executor's execution + # clock to the fixture's now_utc so no test depends on wall time. + with patch.object(joint_site_native, "_observation_clock", side_effect=lambda: _now_utc(base)), patch.object(joint_site_apply, "_clock", side_effect=lambda: _now_utc(base)): + test(base) + print("PASS:", test.__name__) + return False diff --git a/apps/predbat/tests/test_joint_site_arrival.py b/apps/predbat/tests/test_joint_site_arrival.py new file mode 100644 index 000000000..85a61a074 --- /dev/null +++ b/apps/predbat/tests/test_joint_site_arrival.py @@ -0,0 +1,345 @@ +"""Model-only arrival demand cannot become an EV command or a vehicle observation.""" + +from copy import deepcopy +from datetime import datetime, timedelta, timezone +from unittest.mock import patch + +from const import PREDICT_STEP +from joint_site_arrival import prepare_arrival_forecast, model_arrival_slots +from joint_site_native import evaluate_joint_site +from prediction import Prediction +from tests.test_joint_site_actual import _set_entity +from tests.test_joint_site_recurring import recurring_inputs + + +def arrival_inputs(base): + """Give the real native seams an absent car and three genuine historical round trips.""" + (fixture, store) = recurring_inputs(base) + base.car_charging_plan_time = ["07:30:00"] + base.car_charging_limit[0] = base.car_charging_battery_size[0] = 25.0 + base.car_charging_rate[0] = 6.77 + base.forecast_minutes = base.end_record = 1440 + base.car_charging_planned[0] = base.car_charging_now[0] = False + base.car_charging_now_slots = [[] for _ in base.car_charging_slots] + base.joint_site_arrival_forecast = None + base.joint_site_arrival_history = None + start = base.now_utc - timedelta(hours=1) + history = [] + for days, hours in ((3, 2), (2, 3), (1, 4)): + left = start - timedelta(days=days) + for stamp, state in ((left - timedelta(hours=1), "home"), (left, "not_home"), (left + timedelta(minutes=30), "not_home"), (left + timedelta(hours=hours), "home")): + history.append({"state": state, "last_changed": stamp.isoformat()}) + history.append({"state": "not_home", "last_changed": start.isoformat()}) + settings = base.args["joint_site"] + _set_entity(base, settings["home_entity"], "not_home", last_changed=start, last_updated=start) + _set_entity(base, settings["connection_entity"], "disconnected", attributes={"retrievalstatus": "VALID"}) + return fixture, history + + +def test_arrival_forecast_is_model_only_and_no_soc_read(base): + """An uncertain arrival adds native future demand, but neither an EV slot nor permission.""" + (_, writes, _, _), history = arrival_inputs(base) + before = deepcopy(base.args) + slots = deepcopy(base.car_charging_slots) + getter = base.get_state_wrapper + settings = base.args["joint_site"] + + def guarded(*args, **kwargs): + """Make even an indirect absent-vehicle SOC/meter read fail this acceptance test.""" + entity = kwargs.get("entity_id", args[0] if args else None) + assert entity not in (settings["soc_entity"], "sensor.test_ev_energy", "sensor.test_ev_power") + return getter(*args, **kwargs) + + with patch.object(base, "get_state_wrapper", side_effect=guarded), patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=base.now_utc): + base.fetch_sensor_data_car_planning() + report = base.joint_site_arrival_forecast + assert report["status"] == "anticipated", report + assert report["completed_trips"] == 3 and report["eligible_trips"] == 3 + assert report["soc_assumption_kwh"] == 0 and report["soc_observed_at"] is None + assert report["execution_permission"] is False and report["confidence"] == "low" + model = base.car_charging_slots_model() + assert model[0] and model[0][0]["start"] > base.minutes_now + result = evaluate_joint_site(base, publish=False) + assert result["status"] == "household_fallback" and not result["execution_enabled"] + assert result["arrival_forecast"]["status"] == "anticipated" + assert base.joint_site_pending is None + base.publish_car_plan() + published = base.dashboard_values["binary_sensor." + base.prefix + "_car_charging_slot"] + assert published["state"] == "off" + assert not published["attributes"].get("valid_until") + assert base.args == before and base.car_charging_slots == slots and not writes + assert base.car_charging_soc[0] == 0 + + +def test_arrival_predicted_energy_starts_after_expected_arrival(base): + """Use the real Prediction model to prove hypothetical demand changes only future energy.""" + _, history = arrival_inputs(base) + zeros = {minute: 0.0 for minute in range(0, base.forecast_minutes, PREDICT_STEP)} + load = {minute: PREDICT_STEP / 60 for minute in zeros} + with patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + model = base.car_charging_slots_model() + for active in (False, True): + prediction = Prediction(base, zeros, zeros, load, load, car_charging_slots=model if active else base.car_charging_slots) + prediction.run_prediction([], [], [], [], False, end_record=base.forecast_minutes, save="best") + if active: + assert abs(prediction.final_car_soc[0] - 25) < 0.01 + for stamp, value in prediction.predict_car_soc_time[0].items(): + if datetime.fromisoformat(stamp) < base.midnight_utc + timedelta(minutes=model[0][0]["start"]): + assert value == 0, (stamp, value) + else: + assert prediction.final_car_soc[0] == 0 + assert base.car_charging_slots[0] == [] and base.car_charging_soc[0] == 0 + + +def test_arrival_history_invalid_and_force_updates(base): + """Unknown gaps, unpaired home entries and repeated home reports cannot invent trips.""" + for kind in ("empty", "unknown", "same", "future"): + _, history = arrival_inputs(base) + if kind == "empty": + history = [] + elif kind == "unknown": + for record in history: + if record["state"] == "not_home": + record["state"] = "unavailable" + elif kind == "same": + history = [{**record, "state": "home"} for record in history] + else: + history.append({"state": "home", "last_changed": (base.now_utc + timedelta(hours=1)).isoformat()}) + with patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + assert not model_arrival_slots(base), (kind, base.joint_site_arrival_forecast) + + +def test_arrival_history_cache_and_actual_return(base): + """Reuse recorder history, withdraw assumptions immediately on real arrival or permission loss.""" + _, history = arrival_inputs(base) + with patch.object(base, "get_history_wrapper", return_value=[history]) as reader, patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + prepare_arrival_forecast(base) + assert reader.call_count == 1 + assert reader.call_args.kwargs == {"days": 14, "required": False} + assert model_arrival_slots(base) + _set_entity(base, base.args["joint_site"]["home_entity"], "home") + assert not model_arrival_slots(base) + prepare_arrival_forecast(base) + assert not model_arrival_slots(base) + assert reader.call_count == 1 + _, history = arrival_inputs(base) + with patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + _set_entity(base, base.args["joint_site"]["permission_entity"], "off") + assert not model_arrival_slots(base) + + +def test_arrival_late_shortfall_and_model_copy(base): + """An uncertain late return cannot be labelled guaranteed completion or share command lists.""" + _, history = arrival_inputs(base) + base.car_charging_plan_time = [(base.now_utc + timedelta(hours=4)).strftime("%H:%M:%S")] + base.args["timezone"] = "Europe/London" + with patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + report = base.joint_site_arrival_forecast + assert report["status"] == "anticipated" and report["shortfall_kwh"] > 0, report + assert report["guaranteed_target_met"] is False + slots = base.car_charging_slots_model() + slots[0][0]["kwh"] = 999 + assert model_arrival_slots(base)[0]["kwh"] < 999 and not base.car_charging_slots[0] + + +def test_arrival_disabled_stale_and_manual_inputs(base): + """Legacy, stale, malformed and manually controlled inputs retain their existing semantics.""" + for mode in ("off", "shadow", "manual", "relay_on", "stale", "native_slots"): + _, history = arrival_inputs(base) + with patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + if mode in ("off", "shadow"): + base.args["joint_site"]["mode"] = mode + elif mode == "manual": + base.car_charging_manual_soc[0] = True + elif mode == "relay_on": + _set_entity(base, base.args["joint_site"]["relay_entity"], "on") + elif mode == "stale": + base.joint_site_arrival_forecast["prepared_at"] = (base.now_utc - timedelta(minutes=6)).isoformat() + else: + base.car_charging_slots[0] = [{"start": 0, "end": 10, "kwh": 1}] + assert not model_arrival_slots(base), mode + + +def test_arrival_departure_change_rejects_old_trip_history(base): + """A second departure between cycles cannot silently reuse the previous trip's origin.""" + _, history = arrival_inputs(base) + with patch.object(base, "get_history_wrapper", return_value=[history]) as reader, patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + assert model_arrival_slots(base) + _set_entity(base, base.args["joint_site"]["home_entity"], "not_home", last_changed=base.now_utc, last_updated=base.now_utc) + assert not model_arrival_slots(base) + prepare_arrival_forecast(base) + assert reader.call_count == 2 + assert base.joint_site_arrival_forecast["reason"] == "arrival_history_does_not_cover_departure" + assert not model_arrival_slots(base) + + +def test_arrival_fault_and_deadline_change(base): + """Retained physical faults and changed human intent invalidate the hypothetical demand.""" + _, history = arrival_inputs(base) + with patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + assert model_arrival_slots(base) + base.car_charging_plan_time[0] = "08:30:00" + assert not model_arrival_slots(base) + prepare_arrival_forecast(base) + assert base.joint_site_arrival_forecast["ready_by"].endswith("08:30:00+00:00") + store = base.components.get_component("storage") + store.records[("joint_site", "recurring_session")]["invalid"] = "recurring_meter_contradiction" + prepare_arrival_forecast(base) + assert base.joint_site_arrival_forecast["reason"] == "arrival_policy_fault" + assert not model_arrival_slots(base) + + +def test_arrival_dst_uses_native_calendar_deadline(base): + """Model slot offsets follow UTC duration, while the daily ready time stays local 07:30.""" + from zoneinfo import ZoneInfo + + for stamp in ("2026-03-28T19:00:00+01:00", "2026-10-24T19:00:00+02:00"): + _, history = arrival_inputs(base) + now = datetime.fromisoformat(stamp).astimezone(timezone.utc) + shift = now - base.now_utc + history = [{**item, "last_changed": (datetime.fromisoformat(item["last_changed"]) + shift).isoformat()} for item in history] + base.now_utc = now + base.midnight_utc = now.astimezone(ZoneInfo("Europe/Madrid")).replace(hour=0, minute=0, second=0, microsecond=0) + base.minutes_now = int((now - base.midnight_utc).total_seconds() / 60) + base.args["timezone"] = "Europe/Madrid" + for name, state in (("home_entity", "not_home"), ("permission_entity", "on"), ("relay_entity", "off")): + _set_entity(base, base.args["joint_site"][name], state, last_changed=now - timedelta(hours=1), last_updated=now - timedelta(hours=1)) + base.args["joint_site"]["recurring_session"]["policy_authorised_at"] = (now - timedelta(days=1)).isoformat() + with patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=now): + prepare_arrival_forecast(base) + report = base.joint_site_arrival_forecast + assert report["status"] == "anticipated", report + deadline = datetime.fromisoformat(report["ready_by"]) + assert deadline.astimezone(ZoneInfo("Europe/Madrid")).strftime("%H:%M:%S") == "07:30:00" + end = base.midnight_utc.astimezone(timezone.utc) + timedelta(minutes=model_arrival_slots(base)[0]["end"]) + assert end <= deadline + + +def test_arrival_long_trip_uses_observed_return_clock_fallback(base): + """A long trip uses real daily return times instead of treating short trips as an ETA.""" + _, history = arrival_inputs(base) + left = base.now_utc - timedelta(hours=10) + history[-1]["last_changed"] = left.isoformat() + _set_entity(base, base.args["joint_site"]["home_entity"], "not_home", last_changed=left, last_updated=left) + with patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + report = base.joint_site_arrival_forecast + assert report["status"] == "anticipated" and report["cohort"] == "historical_last_return_clock", report + assert report["eligible_trips"] == 3 and report["confidence"] == "low" + assert datetime.fromisoformat(report["expected_arrival"]) > base.now_utc + + +def run_joint_site_arrival_tests(base): + """Register independent model-only arrival acceptance at the native prediction boundary.""" + from tests.test_joint_site_template import shared_template_available, check_unavailable_template + + test_arrival_household_fallback_debug_escapes_hostile_fields(base) + test_arrival_household_fallback_debug_handles_unavailable_forecast(base) + if not shared_template_available(): + check_unavailable_template(base) + return False + for test in ( + test_arrival_forecast_is_model_only_and_no_soc_read, + test_arrival_predicted_energy_starts_after_expected_arrival, + test_arrival_history_invalid_and_force_updates, + test_arrival_history_cache_and_actual_return, + test_arrival_late_shortfall_and_model_copy, + test_arrival_disabled_stale_and_manual_inputs, + test_arrival_departure_change_rejects_old_trip_history, + test_arrival_fault_and_deadline_change, + test_arrival_dst_uses_native_calendar_deadline, + test_arrival_long_trip_uses_observed_return_clock_fallback, + test_arrival_household_fallback_debug_exposes_forecast, + ): + test(base) + return False + + +def test_arrival_household_fallback_debug_exposes_forecast(base): + """The hidden Show-Debug branch surfaces the low-confidence arrival forecast details.""" + from joint_site_preview import render_joint_site_preview + + _, history = arrival_inputs(base) + with patch.object(base, "get_history_wrapper", return_value=[history]), patch("joint_site_native._observation_clock", return_value=base.now_utc): + prepare_arrival_forecast(base) + report = {"status": "household_fallback", "execution_reason": "ev_unavailable", "arrival_forecast": base.joint_site_arrival_forecast} + html = render_joint_site_preview(report, None) + # Hidden debug span, never the visible mainline. + assert html.startswith('